2024-12-03 04:43:01,816 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 04:43:01,828 main DEBUG Took 0.009653 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 04:43:01,828 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 04:43:01,828 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 04:43:01,829 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 04:43:01,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,838 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 04:43:01,850 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,852 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,852 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,853 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,854 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,854 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,855 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,856 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,857 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,857 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,858 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,858 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,859 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,860 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,860 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,861 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,862 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,862 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,863 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,863 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,864 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,864 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,865 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,865 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 04:43:01,866 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,866 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 04:43:01,868 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 04:43:01,870 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 04:43:01,872 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 04:43:01,872 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 04:43:01,874 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 04:43:01,874 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 04:43:01,885 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 04:43:01,888 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 04:43:01,890 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 04:43:01,891 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 04:43:01,891 main DEBUG createAppenders(={Console}) 2024-12-03 04:43:01,892 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-03 04:43:01,893 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 04:43:01,893 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-03 04:43:01,894 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 04:43:01,895 main DEBUG OutputStream closed 2024-12-03 04:43:01,895 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 04:43:01,896 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 04:43:01,896 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-03 04:43:01,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 04:43:01,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 04:43:01,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 04:43:01,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 04:43:01,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 04:43:01,999 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 04:43:01,999 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 04:43:02,000 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 04:43:02,000 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 04:43:02,001 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 04:43:02,001 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 04:43:02,001 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 04:43:02,002 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 04:43:02,002 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 04:43:02,002 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 04:43:02,002 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 04:43:02,003 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 04:43:02,003 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 04:43:02,006 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 04:43:02,007 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-03 04:43:02,007 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 04:43:02,008 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-03T04:43:02,308 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb 2024-12-03 04:43:02,312 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 04:43:02,313 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T04:43:02,325 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-03T04:43:02,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T04:43:02,368 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/cluster_42193a68-94b8-0488-849e-db0fb1205e79, deleteOnExit=true 2024-12-03T04:43:02,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T04:43:02,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/test.cache.data in system properties and HBase conf 2024-12-03T04:43:02,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T04:43:02,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/hadoop.log.dir in system properties and HBase conf 2024-12-03T04:43:02,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T04:43:02,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T04:43:02,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T04:43:02,456 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T04:43:02,547 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T04:43:02,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T04:43:02,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T04:43:02,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T04:43:02,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T04:43:02,552 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T04:43:02,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T04:43:02,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T04:43:02,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T04:43:02,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T04:43:02,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/nfs.dump.dir in system properties and HBase conf 2024-12-03T04:43:02,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/java.io.tmpdir in system properties and HBase conf 2024-12-03T04:43:02,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T04:43:02,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T04:43:02,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T04:43:03,404 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T04:43:03,476 INFO [Time-limited test {}] log.Log(170): Logging initialized @2436ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T04:43:03,550 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T04:43:03,608 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T04:43:03,628 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T04:43:03,628 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T04:43:03,629 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T04:43:03,641 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T04:43:03,643 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/hadoop.log.dir/,AVAILABLE} 2024-12-03T04:43:03,644 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T04:43:03,842 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/java.io.tmpdir/jetty-localhost-37223-hadoop-hdfs-3_4_1-tests_jar-_-any-14259448679111344646/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T04:43:03,849 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37223} 2024-12-03T04:43:03,849 INFO [Time-limited test {}] server.Server(415): Started @2810ms 2024-12-03T04:43:04,292 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T04:43:04,298 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T04:43:04,299 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T04:43:04,299 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T04:43:04,299 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T04:43:04,300 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/hadoop.log.dir/,AVAILABLE} 2024-12-03T04:43:04,301 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T04:43:04,423 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/java.io.tmpdir/jetty-localhost-39909-hadoop-hdfs-3_4_1-tests_jar-_-any-7463370932859861881/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T04:43:04,424 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:39909} 2024-12-03T04:43:04,424 INFO [Time-limited test {}] server.Server(415): Started @3385ms 2024-12-03T04:43:04,480 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T04:43:04,953 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/cluster_42193a68-94b8-0488-849e-db0fb1205e79/data/data1/current/BP-1463157627-172.17.0.3-1733200983164/current, will proceed with Du for space computation calculation, 2024-12-03T04:43:04,962 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/cluster_42193a68-94b8-0488-849e-db0fb1205e79/data/data2/current/BP-1463157627-172.17.0.3-1733200983164/current, will proceed with Du for space computation calculation, 2024-12-03T04:43:05,066 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T04:43:05,147 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x677cedac3a77fd8a with lease ID 0x2f0faf14082429d4: Processing first storage report for DS-93913c6b-9554-4689-b67c-acad18d197e6 from datanode DatanodeRegistration(127.0.0.1:45017, datanodeUuid=0b750991-f791-487f-8920-7516e0a87234, infoPort=35601, infoSecurePort=0, ipcPort=44347, storageInfo=lv=-57;cid=testClusterID;nsid=607990648;c=1733200983164) 2024-12-03T04:43:05,150 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x677cedac3a77fd8a with lease ID 0x2f0faf14082429d4: from storage DS-93913c6b-9554-4689-b67c-acad18d197e6 node DatanodeRegistration(127.0.0.1:45017, datanodeUuid=0b750991-f791-487f-8920-7516e0a87234, infoPort=35601, infoSecurePort=0, ipcPort=44347, storageInfo=lv=-57;cid=testClusterID;nsid=607990648;c=1733200983164), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-03T04:43:05,150 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x677cedac3a77fd8a with lease ID 0x2f0faf14082429d4: Processing first storage report for DS-21df0bd7-eecb-4d50-985b-77d60e1114a5 from datanode DatanodeRegistration(127.0.0.1:45017, datanodeUuid=0b750991-f791-487f-8920-7516e0a87234, infoPort=35601, infoSecurePort=0, ipcPort=44347, storageInfo=lv=-57;cid=testClusterID;nsid=607990648;c=1733200983164) 2024-12-03T04:43:05,150 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x677cedac3a77fd8a with lease ID 0x2f0faf14082429d4: from storage DS-21df0bd7-eecb-4d50-985b-77d60e1114a5 node DatanodeRegistration(127.0.0.1:45017, datanodeUuid=0b750991-f791-487f-8920-7516e0a87234, infoPort=35601, infoSecurePort=0, ipcPort=44347, storageInfo=lv=-57;cid=testClusterID;nsid=607990648;c=1733200983164), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T04:43:05,151 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb 2024-12-03T04:43:05,242 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/cluster_42193a68-94b8-0488-849e-db0fb1205e79/zookeeper_0, clientPort=56849, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/cluster_42193a68-94b8-0488-849e-db0fb1205e79/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/cluster_42193a68-94b8-0488-849e-db0fb1205e79/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T04:43:05,254 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56849 2024-12-03T04:43:05,264 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:43:05,268 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:43:05,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741825_1001 (size=7) 2024-12-03T04:43:05,898 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676 with version=8 2024-12-03T04:43:05,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/hbase-staging 2024-12-03T04:43:06,014 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T04:43:06,265 INFO [Time-limited test {}] client.ConnectionUtils(128): master/32beccd76b35:0 server-side Connection retries=45 2024-12-03T04:43:06,276 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T04:43:06,276 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T04:43:06,281 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T04:43:06,281 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T04:43:06,281 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T04:43:06,414 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T04:43:06,474 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T04:43:06,482 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T04:43:06,486 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T04:43:06,512 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4687 (auto-detected) 2024-12-03T04:43:06,513 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T04:43:06,532 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38397 2024-12-03T04:43:06,557 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38397 connecting to ZooKeeper ensemble=127.0.0.1:56849 2024-12-03T04:43:06,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:383970x0, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T04:43:06,589 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38397-0x101516731320000 connected 2024-12-03T04:43:06,616 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:43:06,619 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:43:06,627 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T04:43:06,631 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676, hbase.cluster.distributed=false 2024-12-03T04:43:06,655 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T04:43:06,659 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38397 2024-12-03T04:43:06,659 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38397 2024-12-03T04:43:06,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38397 2024-12-03T04:43:06,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38397 2024-12-03T04:43:06,661 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38397 2024-12-03T04:43:06,769 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32beccd76b35:0 server-side Connection retries=45 2024-12-03T04:43:06,770 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T04:43:06,771 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T04:43:06,771 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T04:43:06,771 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T04:43:06,771 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T04:43:06,774 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T04:43:06,776 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T04:43:06,777 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43193 2024-12-03T04:43:06,779 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43193 connecting to ZooKeeper ensemble=127.0.0.1:56849 2024-12-03T04:43:06,780 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:43:06,783 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:43:06,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:431930x0, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T04:43:06,790 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43193-0x101516731320001 connected 2024-12-03T04:43:06,790 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T04:43:06,794 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T04:43:06,802 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T04:43:06,805 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T04:43:06,811 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T04:43:06,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43193 2024-12-03T04:43:06,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43193 2024-12-03T04:43:06,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43193 2024-12-03T04:43:06,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43193 2024-12-03T04:43:06,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43193 2024-12-03T04:43:06,835 DEBUG [M:0;32beccd76b35:38397 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;32beccd76b35:38397 2024-12-03T04:43:06,836 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/32beccd76b35,38397,1733200986071 2024-12-03T04:43:06,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T04:43:06,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T04:43:06,844 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/32beccd76b35,38397,1733200986071 2024-12-03T04:43:06,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T04:43:06,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:06,874 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T04:43:06,876 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/32beccd76b35,38397,1733200986071 from backup master directory 2024-12-03T04:43:06,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:06,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/32beccd76b35,38397,1733200986071 2024-12-03T04:43:06,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T04:43:06,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T04:43:06,883 WARN [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T04:43:06,883 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=32beccd76b35,38397,1733200986071 2024-12-03T04:43:06,886 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T04:43:06,890 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T04:43:06,963 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/hbase.id] with ID: 08870cf8-d835-4c97-9170-0203a4d6da45 2024-12-03T04:43:06,963 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/.tmp/hbase.id 2024-12-03T04:43:06,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741826_1002 (size=42) 2024-12-03T04:43:07,376 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/.tmp/hbase.id]:[hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/hbase.id] 2024-12-03T04:43:07,421 INFO [master/32beccd76b35:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T04:43:07,425 INFO [master/32beccd76b35:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T04:43:07,444 INFO [master/32beccd76b35:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-03T04:43:07,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:07,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:07,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741827_1003 (size=196) 2024-12-03T04:43:07,881 INFO [master/32beccd76b35:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:43:07,884 INFO [master/32beccd76b35:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T04:43:07,905 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:07,909 INFO [master/32beccd76b35:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T04:43:07,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741828_1004 (size=1189) 2024-12-03T04:43:08,362 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store 2024-12-03T04:43:08,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741829_1005 (size=34) 2024-12-03T04:43:08,385 INFO [master/32beccd76b35:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T04:43:08,388 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:43:08,389 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T04:43:08,390 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:43:08,390 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:43:08,391 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T04:43:08,391 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:43:08,391 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:43:08,392 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733200988389Disabling compacts and flushes for region at 1733200988389Disabling writes for close at 1733200988391 (+2 ms)Writing region close event to WAL at 1733200988391Closed at 1733200988391 2024-12-03T04:43:08,394 WARN [master/32beccd76b35:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/.initializing 2024-12-03T04:43:08,395 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/WALs/32beccd76b35,38397,1733200986071 2024-12-03T04:43:08,403 INFO [master/32beccd76b35:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T04:43:08,417 INFO [master/32beccd76b35:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32beccd76b35%2C38397%2C1733200986071, suffix=, logDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/WALs/32beccd76b35,38397,1733200986071, archiveDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/oldWALs, maxLogs=10 2024-12-03T04:43:08,447 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/WALs/32beccd76b35,38397,1733200986071/32beccd76b35%2C38397%2C1733200986071.1733200988423, exclude list is [], retry=0 2024-12-03T04:43:08,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45017,DS-93913c6b-9554-4689-b67c-acad18d197e6,DISK] 2024-12-03T04:43:08,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T04:43:08,500 INFO [master/32beccd76b35:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/WALs/32beccd76b35,38397,1733200986071/32beccd76b35%2C38397%2C1733200986071.1733200988423 2024-12-03T04:43:08,501 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35601:35601)] 2024-12-03T04:43:08,502 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:43:08,502 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:43:08,505 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,506 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,542 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,573 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T04:43:08,577 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:08,580 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:43:08,580 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T04:43:08,584 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:08,585 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:08,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T04:43:08,589 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:08,590 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:08,590 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T04:43:08,592 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:08,593 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:08,594 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,598 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,600 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,606 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,607 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,610 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T04:43:08,614 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T04:43:08,618 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T04:43:08,619 INFO [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75457284, jitterRate=0.12440115213394165}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T04:43:08,624 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733200988518Initializing all the Stores at 1733200988520 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733200988521 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733200988522 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733200988522Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733200988522Cleaning up temporary data from old regions at 1733200988607 (+85 ms)Region opened successfully at 1733200988624 (+17 ms) 2024-12-03T04:43:08,625 INFO [master/32beccd76b35:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T04:43:08,658 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27857d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32beccd76b35/172.17.0.3:0 2024-12-03T04:43:08,690 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T04:43:08,702 INFO [master/32beccd76b35:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T04:43:08,702 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T04:43:08,705 INFO [master/32beccd76b35:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T04:43:08,707 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T04:43:08,712 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T04:43:08,712 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T04:43:08,740 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T04:43:08,750 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T04:43:08,752 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T04:43:08,755 INFO [master/32beccd76b35:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T04:43:08,756 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T04:43:08,758 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T04:43:08,760 INFO [master/32beccd76b35:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T04:43:08,764 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T04:43:08,765 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T04:43:08,767 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T04:43:08,768 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T04:43:08,786 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T04:43:08,787 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T04:43:08,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T04:43:08,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T04:43:08,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:08,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:08,795 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=32beccd76b35,38397,1733200986071, sessionid=0x101516731320000, setting cluster-up flag (Was=false) 2024-12-03T04:43:08,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:08,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:08,813 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T04:43:08,814 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32beccd76b35,38397,1733200986071 2024-12-03T04:43:08,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:08,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:08,825 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T04:43:08,827 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32beccd76b35,38397,1733200986071 2024-12-03T04:43:08,833 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T04:43:08,913 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T04:43:08,923 INFO [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(746): ClusterId : 08870cf8-d835-4c97-9170-0203a4d6da45 2024-12-03T04:43:08,924 INFO [master/32beccd76b35:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T04:43:08,927 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T04:43:08,930 INFO [master/32beccd76b35:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T04:43:08,932 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T04:43:08,932 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T04:43:08,936 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T04:43:08,937 DEBUG [RS:0;32beccd76b35:43193 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd41370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32beccd76b35/172.17.0.3:0 2024-12-03T04:43:08,937 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 32beccd76b35,38397,1733200986071 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T04:43:08,946 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/32beccd76b35:0, corePoolSize=5, maxPoolSize=5 2024-12-03T04:43:08,947 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/32beccd76b35:0, corePoolSize=5, maxPoolSize=5 2024-12-03T04:43:08,947 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/32beccd76b35:0, corePoolSize=5, maxPoolSize=5 2024-12-03T04:43:08,947 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/32beccd76b35:0, corePoolSize=5, maxPoolSize=5 2024-12-03T04:43:08,947 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/32beccd76b35:0, corePoolSize=10, maxPoolSize=10 2024-12-03T04:43:08,948 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:08,948 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/32beccd76b35:0, corePoolSize=2, maxPoolSize=2 2024-12-03T04:43:08,948 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:08,954 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T04:43:08,955 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T04:43:08,957 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733201018957 2024-12-03T04:43:08,958 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;32beccd76b35:43193 2024-12-03T04:43:08,960 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T04:43:08,961 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T04:43:08,962 INFO [RS:0;32beccd76b35:43193 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T04:43:08,962 INFO [RS:0;32beccd76b35:43193 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T04:43:08,962 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T04:43:08,963 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:08,963 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T04:43:08,966 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T04:43:08,966 INFO [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(2659): reportForDuty to master=32beccd76b35,38397,1733200986071 with port=43193, startcode=1733200986730 2024-12-03T04:43:08,966 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T04:43:08,967 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T04:43:08,967 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T04:43:08,978 DEBUG [RS:0;32beccd76b35:43193 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T04:43:08,979 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:08,986 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T04:43:08,988 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T04:43:08,988 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T04:43:08,991 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T04:43:08,991 INFO [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T04:43:08,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741831_1007 (size=1321) 2024-12-03T04:43:08,998 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/32beccd76b35:0:becomeActiveMaster-HFileCleaner.large.0-1733200988993,5,FailOnTimeoutGroup] 2024-12-03T04:43:08,999 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T04:43:09,000 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676 2024-12-03T04:43:09,001 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/32beccd76b35:0:becomeActiveMaster-HFileCleaner.small.0-1733200988998,5,FailOnTimeoutGroup] 2024-12-03T04:43:09,002 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,002 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T04:43:09,003 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,004 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741832_1008 (size=32) 2024-12-03T04:43:09,033 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56273, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T04:43:09,039 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38397 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32beccd76b35,43193,1733200986730 2024-12-03T04:43:09,042 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38397 {}] master.ServerManager(517): Registering regionserver=32beccd76b35,43193,1733200986730 2024-12-03T04:43:09,057 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676 2024-12-03T04:43:09,057 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44839 2024-12-03T04:43:09,057 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T04:43:09,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T04:43:09,062 DEBUG [RS:0;32beccd76b35:43193 {}] zookeeper.ZKUtil(111): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32beccd76b35,43193,1733200986730 2024-12-03T04:43:09,062 WARN [RS:0;32beccd76b35:43193 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T04:43:09,062 INFO [RS:0;32beccd76b35:43193 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T04:43:09,062 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/WALs/32beccd76b35,43193,1733200986730 2024-12-03T04:43:09,064 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32beccd76b35,43193,1733200986730] 2024-12-03T04:43:09,087 INFO [RS:0;32beccd76b35:43193 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T04:43:09,100 INFO [RS:0;32beccd76b35:43193 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T04:43:09,105 INFO [RS:0;32beccd76b35:43193 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T04:43:09,106 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,107 INFO [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T04:43:09,113 INFO [RS:0;32beccd76b35:43193 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T04:43:09,114 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,115 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:09,115 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:09,115 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:09,115 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:09,116 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:09,116 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32beccd76b35:0, corePoolSize=2, maxPoolSize=2 2024-12-03T04:43:09,116 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:09,116 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:09,116 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:09,116 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:09,117 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:09,117 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32beccd76b35:0, corePoolSize=1, maxPoolSize=1 2024-12-03T04:43:09,117 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32beccd76b35:0, corePoolSize=3, maxPoolSize=3 2024-12-03T04:43:09,117 DEBUG [RS:0;32beccd76b35:43193 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0, corePoolSize=3, maxPoolSize=3 2024-12-03T04:43:09,118 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,118 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,118 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,118 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,119 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,119 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,43193,1733200986730-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T04:43:09,142 INFO [RS:0;32beccd76b35:43193 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T04:43:09,145 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,43193,1733200986730-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,145 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,146 INFO [RS:0;32beccd76b35:43193 {}] regionserver.Replication(171): 32beccd76b35,43193,1733200986730 started 2024-12-03T04:43:09,171 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:09,171 INFO [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1482): Serving as 32beccd76b35,43193,1733200986730, RpcServer on 32beccd76b35/172.17.0.3:43193, sessionid=0x101516731320001 2024-12-03T04:43:09,172 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T04:43:09,172 DEBUG [RS:0;32beccd76b35:43193 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32beccd76b35,43193,1733200986730 2024-12-03T04:43:09,173 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32beccd76b35,43193,1733200986730' 2024-12-03T04:43:09,173 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T04:43:09,174 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T04:43:09,175 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T04:43:09,175 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T04:43:09,176 DEBUG [RS:0;32beccd76b35:43193 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32beccd76b35,43193,1733200986730 2024-12-03T04:43:09,176 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32beccd76b35,43193,1733200986730' 2024-12-03T04:43:09,176 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T04:43:09,178 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T04:43:09,178 DEBUG [RS:0;32beccd76b35:43193 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T04:43:09,178 INFO [RS:0;32beccd76b35:43193 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T04:43:09,178 INFO [RS:0;32beccd76b35:43193 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T04:43:09,284 INFO [RS:0;32beccd76b35:43193 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T04:43:09,287 INFO [RS:0;32beccd76b35:43193 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32beccd76b35%2C43193%2C1733200986730, suffix=, logDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/WALs/32beccd76b35,43193,1733200986730, archiveDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/oldWALs, maxLogs=32 2024-12-03T04:43:09,304 DEBUG [RS:0;32beccd76b35:43193 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/WALs/32beccd76b35,43193,1733200986730/32beccd76b35%2C43193%2C1733200986730.1733200989290, exclude list is [], retry=0 2024-12-03T04:43:09,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45017,DS-93913c6b-9554-4689-b67c-acad18d197e6,DISK] 2024-12-03T04:43:09,311 INFO [RS:0;32beccd76b35:43193 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/WALs/32beccd76b35,43193,1733200986730/32beccd76b35%2C43193%2C1733200986730.1733200989290 2024-12-03T04:43:09,312 DEBUG [RS:0;32beccd76b35:43193 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35601:35601)] 2024-12-03T04:43:09,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:43:09,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T04:43:09,426 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T04:43:09,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:09,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:43:09,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T04:43:09,430 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T04:43:09,430 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:09,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:43:09,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T04:43:09,434 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T04:43:09,434 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:09,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:43:09,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T04:43:09,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T04:43:09,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:09,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:43:09,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T04:43:09,441 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740 2024-12-03T04:43:09,441 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740 2024-12-03T04:43:09,444 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T04:43:09,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T04:43:09,446 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:43:09,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T04:43:09,452 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T04:43:09,453 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65679892, jitterRate=-0.021293342113494873}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:43:09,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733200989422Initializing all the Stores at 1733200989423 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733200989424 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733200989424Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733200989424Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733200989424Cleaning up temporary data from old regions at 1733200989445 (+21 ms)Region opened successfully at 1733200989455 (+10 ms) 2024-12-03T04:43:09,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T04:43:09,456 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T04:43:09,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T04:43:09,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T04:43:09,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T04:43:09,458 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T04:43:09,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733200989456Disabling compacts and flushes for region at 1733200989456Disabling writes for close at 1733200989456Writing region close event to WAL at 1733200989457 (+1 ms)Closed at 1733200989458 (+1 ms) 2024-12-03T04:43:09,462 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T04:43:09,462 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T04:43:09,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T04:43:09,477 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T04:43:09,479 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T04:43:09,632 DEBUG [32beccd76b35:38397 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T04:43:09,647 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:43:09,655 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32beccd76b35,43193,1733200986730, state=OPENING 2024-12-03T04:43:09,662 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T04:43:09,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:09,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:43:09,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T04:43:09,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T04:43:09,668 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T04:43:09,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:43:09,845 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T04:43:09,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41201, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T04:43:09,860 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T04:43:09,861 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T04:43:09,861 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T04:43:09,865 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32beccd76b35%2C43193%2C1733200986730.meta, suffix=.meta, logDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/WALs/32beccd76b35,43193,1733200986730, archiveDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/oldWALs, maxLogs=32 2024-12-03T04:43:09,881 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/WALs/32beccd76b35,43193,1733200986730/32beccd76b35%2C43193%2C1733200986730.meta.1733200989867.meta, exclude list is [], retry=0 2024-12-03T04:43:09,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45017,DS-93913c6b-9554-4689-b67c-acad18d197e6,DISK] 2024-12-03T04:43:09,888 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/WALs/32beccd76b35,43193,1733200986730/32beccd76b35%2C43193%2C1733200986730.meta.1733200989867.meta 2024-12-03T04:43:09,888 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35601:35601)] 2024-12-03T04:43:09,889 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:43:09,891 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T04:43:09,893 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T04:43:09,898 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T04:43:09,902 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T04:43:09,903 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:43:09,903 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T04:43:09,903 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T04:43:09,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T04:43:09,908 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T04:43:09,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:09,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:43:09,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T04:43:09,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T04:43:09,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:09,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:43:09,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T04:43:09,913 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T04:43:09,914 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:09,914 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:43:09,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T04:43:09,916 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T04:43:09,916 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:09,917 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T04:43:09,918 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T04:43:09,922 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740 2024-12-03T04:43:09,928 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740 2024-12-03T04:43:09,930 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T04:43:09,930 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T04:43:09,932 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:43:09,935 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T04:43:09,937 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68311482, jitterRate=0.01792040467262268}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:43:09,937 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T04:43:09,939 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733200989904Writing region info on filesystem at 1733200989904Initializing all the Stores at 1733200989906 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733200989906Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733200989906Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733200989906Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733200989906Cleaning up temporary data from old regions at 1733200989931 (+25 ms)Running coprocessor post-open hooks at 1733200989937 (+6 ms)Region opened successfully at 1733200989939 (+2 ms) 2024-12-03T04:43:09,948 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733200989836 2024-12-03T04:43:09,962 DEBUG [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T04:43:09,963 INFO [RS_OPEN_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T04:43:09,964 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:43:09,967 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32beccd76b35,43193,1733200986730, state=OPEN 2024-12-03T04:43:09,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T04:43:09,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T04:43:09,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T04:43:09,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T04:43:09,972 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:09,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T04:43:09,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32beccd76b35,43193,1733200986730 in 303 msec 2024-12-03T04:43:09,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T04:43:09,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 512 msec 2024-12-03T04:43:09,988 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T04:43:09,988 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T04:43:10,013 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:10,015 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:10,044 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:10,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34455, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:10,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2150 sec 2024-12-03T04:43:10,077 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733200990077, completionTime=-1 2024-12-03T04:43:10,080 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T04:43:10,080 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T04:43:10,107 INFO [master/32beccd76b35:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T04:43:10,107 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733201050107 2024-12-03T04:43:10,107 INFO [master/32beccd76b35:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733201110107 2024-12-03T04:43:10,107 INFO [master/32beccd76b35:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-12-03T04:43:10,110 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,38397,1733200986071-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:10,110 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,38397,1733200986071-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:10,111 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,38397,1733200986071-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:10,112 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-32beccd76b35:38397, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:10,112 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:10,113 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:10,118 DEBUG [master/32beccd76b35:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T04:43:10,160 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.277sec 2024-12-03T04:43:10,161 INFO [master/32beccd76b35:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T04:43:10,163 INFO [master/32beccd76b35:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T04:43:10,164 INFO [master/32beccd76b35:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T04:43:10,164 INFO [master/32beccd76b35:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T04:43:10,165 INFO [master/32beccd76b35:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T04:43:10,166 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,38397,1733200986071-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T04:43:10,166 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,38397,1733200986071-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T04:43:10,179 DEBUG [master/32beccd76b35:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T04:43:10,180 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T04:43:10,180 INFO [master/32beccd76b35:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32beccd76b35,38397,1733200986071-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T04:43:10,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:10,239 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T04:43:10,239 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T04:43:10,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:10,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:10,254 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3df85c5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:10,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:10,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:10,264 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:10,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:10,268 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:10,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:10,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=32beccd76b35,38397,1733200986071 2024-12-03T04:43:10,314 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=209, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=298, ProcessCount=11, AvailableMemoryMB=4661 2024-12-03T04:43:10,323 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T04:43:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,38397,1733200986071 2024-12-03T04:43:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f9587a2 2024-12-03T04:43:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T04:43:10,334 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T04:43:10,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T04:43:10,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:43:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T04:43:10,350 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T04:43:10,353 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:10,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-03T04:43:10,355 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T04:43:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T04:43:10,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741835_1011 (size=1061) 2024-12-03T04:43:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T04:43:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T04:43:10,773 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676 2024-12-03T04:43:10,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741836_1012 (size=53) 2024-12-03T04:43:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T04:43:11,183 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:43:11,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2444c4148e3a1c831bc6f992044b0533, disabling compactions & flushes 2024-12-03T04:43:11,184 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:11,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:11,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. after waiting 0 ms 2024-12-03T04:43:11,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:11,184 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:11,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2444c4148e3a1c831bc6f992044b0533: Waiting for close lock at 1733200991184Disabling compacts and flushes for region at 1733200991184Disabling writes for close at 1733200991184Writing region close event to WAL at 1733200991184Closed at 1733200991184 2024-12-03T04:43:11,187 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T04:43:11,192 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733200991187"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733200991187"}]},"ts":"1733200991187"} 2024-12-03T04:43:11,197 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T04:43:11,200 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T04:43:11,203 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733200991200"}]},"ts":"1733200991200"} 2024-12-03T04:43:11,207 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T04:43:11,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2444c4148e3a1c831bc6f992044b0533, ASSIGN}] 2024-12-03T04:43:11,212 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2444c4148e3a1c831bc6f992044b0533, ASSIGN 2024-12-03T04:43:11,214 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2444c4148e3a1c831bc6f992044b0533, ASSIGN; state=OFFLINE, location=32beccd76b35,43193,1733200986730; forceNewPlan=false, retain=false 2024-12-03T04:43:11,366 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2444c4148e3a1c831bc6f992044b0533, regionState=OPENING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:43:11,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2444c4148e3a1c831bc6f992044b0533, ASSIGN because future has completed 2024-12-03T04:43:11,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:43:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T04:43:11,532 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:11,532 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:43:11,532 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,533 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:43:11,533 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,533 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,535 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,538 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:43:11,539 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2444c4148e3a1c831bc6f992044b0533 columnFamilyName A 2024-12-03T04:43:11,539 DEBUG [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:11,540 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] regionserver.HStore(327): Store=2444c4148e3a1c831bc6f992044b0533/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:11,540 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,542 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:43:11,542 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2444c4148e3a1c831bc6f992044b0533 columnFamilyName B 2024-12-03T04:43:11,542 DEBUG [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:11,543 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] regionserver.HStore(327): Store=2444c4148e3a1c831bc6f992044b0533/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:11,543 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,545 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:43:11,545 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2444c4148e3a1c831bc6f992044b0533 columnFamilyName C 2024-12-03T04:43:11,545 DEBUG [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:11,546 INFO [StoreOpener-2444c4148e3a1c831bc6f992044b0533-1 {}] regionserver.HStore(327): Store=2444c4148e3a1c831bc6f992044b0533/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:11,547 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:11,547 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,548 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,549 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,551 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,551 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,552 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:43:11,554 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,558 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T04:43:11,559 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2444c4148e3a1c831bc6f992044b0533; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67746947, jitterRate=0.009508177638053894}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:43:11,559 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:11,561 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2444c4148e3a1c831bc6f992044b0533: Running coprocessor pre-open hook at 1733200991533Writing region info on filesystem at 1733200991533Initializing all the Stores at 1733200991535 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733200991535Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733200991535Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733200991535Cleaning up temporary data from old regions at 1733200991551 (+16 ms)Running coprocessor post-open hooks at 1733200991559 (+8 ms)Region opened successfully at 1733200991561 (+2 ms) 2024-12-03T04:43:11,563 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., pid=6, masterSystemTime=1733200991525 2024-12-03T04:43:11,567 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:11,567 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:11,568 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2444c4148e3a1c831bc6f992044b0533, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:43:11,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:43:11,578 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T04:43:11,578 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 in 203 msec 2024-12-03T04:43:11,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T04:43:11,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2444c4148e3a1c831bc6f992044b0533, ASSIGN in 368 msec 2024-12-03T04:43:11,584 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T04:43:11,585 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733200991584"}]},"ts":"1733200991584"} 2024-12-03T04:43:11,588 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T04:43:11,590 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T04:43:11,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2470 sec 2024-12-03T04:43:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T04:43:12,523 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:12,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705975cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:12,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:12,532 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d6336a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:12,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:12,535 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7323ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,538 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:12,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:12,547 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T04:43:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,38397,1733200986071 2024-12-03T04:43:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58b5dcfd 2024-12-03T04:43:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T04:43:12,553 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T04:43:12,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6023f6f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:12,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:12,558 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703ed473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:12,561 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:12,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:12,565 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:12,568 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:12,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bed14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:12,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:12,572 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:12,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:12,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af2c0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:12,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:12,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:12,575 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612488f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:12,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:12,577 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9dd8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:12,580 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3ac293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:12,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:12,582 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fe47336, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:12,584 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adf4095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,587 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:12,587 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:12,588 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:12,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e154f6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:12,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:12,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:12,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c8578f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:12,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:12,592 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24aa1c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:12,594 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fce0532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:12,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:12,597 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee22a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:12,599 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23451635, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:12,600 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:12,601 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83f8e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:12,603 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c944594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:12,607 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:12,607 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:12,607 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:12,608 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:12,609 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:12,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:12,609 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:12,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:12,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:12,610 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:12,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:12,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:12,612 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:12,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:12,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:12,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:12,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:12,616 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:12,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:12,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-03T04:43:12,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:12,620 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:43:12,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:43:12,624 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:12,624 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:12,626 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:43:12,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:43:12,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:43:12,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:43:12,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:43:12,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:43:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:43:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:43:12,634 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:12,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:12,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:43:12,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:12,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:43:12,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:43:12,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:12,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:12,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:12,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:12,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:12,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:12,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/e828cac9104947cfad9853660faee996 is 50, key is test_row_0/A:col10/1733200992695/Put/seqid=0 2024-12-03T04:43:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201052800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:12,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201052813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:12,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201052817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:12,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201052822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:12,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733200992803, max timestamp=1733200992817Num uniques 859; 2024-12-03T04:43:12,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:12,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733200992803, max timestamp=1733200992817Num uniques 859; 2024-12-03T04:43:12,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:12,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733200992803, max timestamp=1733200992817Num uniques 859; 2024-12-03T04:43:12,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:12,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741837_1013 (size=12001) 2024-12-03T04:43:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:43:13,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:43:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:13,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:13,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:13,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201053036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201053033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201053048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201053052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201053053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:43:13,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201053163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201053163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:13,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:13,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:13,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201053175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201053175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201053177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:43:13,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/e828cac9104947cfad9853660faee996 2024-12-03T04:43:13,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:43:13,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:13,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:13,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:13,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201053383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201053386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201053393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201053394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201053402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/840029dc1a774ce4bd69de492f93cfdd is 50, key is test_row_0/B:col10/1733200992695/Put/seqid=0 2024-12-03T04:43:13,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:43:13,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741838_1014 (size=12001) 2024-12-03T04:43:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:13,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/840029dc1a774ce4bd69de492f93cfdd 2024-12-03T04:43:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:13,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/a129253573a44b3eababd773271be6f9 is 50, key is test_row_0/C:col10/1733200992695/Put/seqid=0 2024-12-03T04:43:13,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741839_1015 (size=12001) 2024-12-03T04:43:13,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/a129253573a44b3eababd773271be6f9 2024-12-03T04:43:13,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/e828cac9104947cfad9853660faee996 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/e828cac9104947cfad9853660faee996 2024-12-03T04:43:13,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/e828cac9104947cfad9853660faee996, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T04:43:13,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/840029dc1a774ce4bd69de492f93cfdd as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/840029dc1a774ce4bd69de492f93cfdd 2024-12-03T04:43:13,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/840029dc1a774ce4bd69de492f93cfdd, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T04:43:13,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/a129253573a44b3eababd773271be6f9 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a129253573a44b3eababd773271be6f9 2024-12-03T04:43:13,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a129253573a44b3eababd773271be6f9, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T04:43:13,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 2444c4148e3a1c831bc6f992044b0533 in 947ms, sequenceid=13, compaction requested=false 2024-12-03T04:43:13,655 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T04:43:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:13,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T04:43:13,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:13,665 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-12-03T04:43:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:13,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:13,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:13,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:13,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:13,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/a786a70b41a24a90b6c903e3ae30b67c is 50, key is test_row_0/A:col10/1733200993043/Put/seqid=0 2024-12-03T04:43:13,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741840_1016 (size=12001) 2024-12-03T04:43:13,703 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/a786a70b41a24a90b6c903e3ae30b67c 2024-12-03T04:43:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:13,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:13,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/b451529e47bb46f1b2e7b452dd5a8be7 is 50, key is test_row_0/B:col10/1733200993043/Put/seqid=0 2024-12-03T04:43:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:43:13,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741841_1017 (size=12001) 2024-12-03T04:43:13,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201053765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201053790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201053799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201053821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201053821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201053933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201053945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201053946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201053963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201053963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:14,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201054158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:14,179 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/b451529e47bb46f1b2e7b452dd5a8be7 2024-12-03T04:43:14,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201054174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:14,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:14,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201054183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:14,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201054183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:14,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201054184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:14,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:14,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/f755edb8af9b411db220f4cd96b4fb17 is 50, key is test_row_0/C:col10/1733200993043/Put/seqid=0 2024-12-03T04:43:14,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741842_1018 (size=12001) 2024-12-03T04:43:14,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201054472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201054492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:14,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201054502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:14,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201054503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:14,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201054503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:14,649 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/f755edb8af9b411db220f4cd96b4fb17 2024-12-03T04:43:14,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/a786a70b41a24a90b6c903e3ae30b67c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a786a70b41a24a90b6c903e3ae30b67c 2024-12-03T04:43:14,685 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a786a70b41a24a90b6c903e3ae30b67c, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T04:43:14,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/b451529e47bb46f1b2e7b452dd5a8be7 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/b451529e47bb46f1b2e7b452dd5a8be7 2024-12-03T04:43:14,706 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/b451529e47bb46f1b2e7b452dd5a8be7, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T04:43:14,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/f755edb8af9b411db220f4cd96b4fb17 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f755edb8af9b411db220f4cd96b4fb17 2024-12-03T04:43:14,721 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f755edb8af9b411db220f4cd96b4fb17, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T04:43:14,729 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 2444c4148e3a1c831bc6f992044b0533 in 1064ms, sequenceid=48, compaction requested=false 2024-12-03T04:43:14,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:14,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:14,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-03T04:43:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-03T04:43:14,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T04:43:14,753 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1050 sec 2024-12-03T04:43:14,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.1450 sec 2024-12-03T04:43:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T04:43:14,778 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:14,785 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-03T04:43:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T04:43:14,802 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:14,810 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:14,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T04:43:14,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T04:43:14,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:14,989 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:43:14,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:14,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:14,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:14,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:14,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:14,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:14,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:15,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/6bb91d2038da4f55832e964d463d743d is 50, key is test_row_0/A:col10/1733200993789/Put/seqid=0 2024-12-03T04:43:15,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741843_1019 (size=12001) 2024-12-03T04:43:15,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201055104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201055106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201055112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201055113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T04:43:15,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733200995092, max timestamp=1733200995107Num uniques 859; 2024-12-03T04:43:15,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:15,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733200995092, max timestamp=1733200995107Num uniques 859; 2024-12-03T04:43:15,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:15,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733200995092, max timestamp=1733200995107Num uniques 859; 2024-12-03T04:43:15,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:15,423 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T04:43:15,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201055438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,470 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/6bb91d2038da4f55832e964d463d743d 2024-12-03T04:43:15,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201055443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201055451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201055440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201055453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T04:43:15,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/569f45e390c94bbd8900a3b207df3b95 is 50, key is test_row_0/B:col10/1733200993789/Put/seqid=0 2024-12-03T04:43:15,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741844_1020 (size=12001) 2024-12-03T04:43:15,556 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T04:43:15,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201055572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201055582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201055583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201055583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201055583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201055792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201055803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201055803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:15,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201055806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201055804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:15,927 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/569f45e390c94bbd8900a3b207df3b95 2024-12-03T04:43:15,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/435f7f0032af4c4697f5a932f0ccba6c is 50, key is test_row_0/C:col10/1733200993789/Put/seqid=0 2024-12-03T04:43:16,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741845_1021 (size=12001) 2024-12-03T04:43:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T04:43:16,003 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/435f7f0032af4c4697f5a932f0ccba6c 2024-12-03T04:43:16,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/6bb91d2038da4f55832e964d463d743d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6bb91d2038da4f55832e964d463d743d 2024-12-03T04:43:16,043 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6bb91d2038da4f55832e964d463d743d, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T04:43:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/569f45e390c94bbd8900a3b207df3b95 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/569f45e390c94bbd8900a3b207df3b95 2024-12-03T04:43:16,069 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/569f45e390c94bbd8900a3b207df3b95, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T04:43:16,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/435f7f0032af4c4697f5a932f0ccba6c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/435f7f0032af4c4697f5a932f0ccba6c 2024-12-03T04:43:16,098 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/435f7f0032af4c4697f5a932f0ccba6c, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T04:43:16,100 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 2444c4148e3a1c831bc6f992044b0533 in 1111ms, sequenceid=60, compaction requested=true 2024-12-03T04:43:16,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-03T04:43:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-03T04:43:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:16,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=134.18 KB heapSize=385.22 KB 2024-12-03T04:43:16,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-03T04:43:16,137 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2990 sec 2024-12-03T04:43:16,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.3520 sec 2024-12-03T04:43:16,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733200995437, max timestamp=1733200996129Num uniques 1018; 2024-12-03T04:43:16,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T04:43:16,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733200995437, max timestamp=1733200996129Num uniques 1018; 2024-12-03T04:43:16,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T04:43:16,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:16,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:16,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:16,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:16,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:16,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:43:16,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/d0c2819fd75844ab95d49a67ff1b7deb is 50, key is test_row_0/A:col10/1733200995437/Put/seqid=0 2024-12-03T04:43:16,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741846_1022 (size=12001) 2024-12-03T04:43:16,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/d0c2819fd75844ab95d49a67ff1b7deb 2024-12-03T04:43:16,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/88ea2bec57184cac8f58d6bd5d04f11a is 50, key is test_row_0/B:col10/1733200995437/Put/seqid=0 2024-12-03T04:43:16,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741847_1023 (size=12001) 2024-12-03T04:43:16,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201056269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201056272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201056277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733200996227, max timestamp=1733200996279Num uniques 859; 2024-12-03T04:43:16,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:16,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733200996264, max timestamp=1733200996279Num uniques 859; 2024-12-03T04:43:16,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T04:43:16,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201056344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201056348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201056391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201056404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201056405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T04:43:16,472 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T04:43:16,476 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T04:43:16,476 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T04:43:16,476 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T04:43:16,476 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T04:43:16,478 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T04:43:16,478 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T04:43:16,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201056474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201056474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201056606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201056613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201056630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/88ea2bec57184cac8f58d6bd5d04f11a 2024-12-03T04:43:16,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201056693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201056693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/94c2922d75204c36a9f0bc090791b5ed is 50, key is test_row_0/C:col10/1733200995437/Put/seqid=0 2024-12-03T04:43:16,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741848_1024 (size=12001) 2024-12-03T04:43:16,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/94c2922d75204c36a9f0bc090791b5ed 2024-12-03T04:43:16,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/d0c2819fd75844ab95d49a67ff1b7deb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/d0c2819fd75844ab95d49a67ff1b7deb 2024-12-03T04:43:16,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/d0c2819fd75844ab95d49a67ff1b7deb, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T04:43:16,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/88ea2bec57184cac8f58d6bd5d04f11a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88ea2bec57184cac8f58d6bd5d04f11a 2024-12-03T04:43:16,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201056934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201056942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201056951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:16,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88ea2bec57184cac8f58d6bd5d04f11a, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T04:43:16,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/94c2922d75204c36a9f0bc090791b5ed as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/94c2922d75204c36a9f0bc090791b5ed 2024-12-03T04:43:17,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/94c2922d75204c36a9f0bc090791b5ed, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T04:43:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T04:43:17,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~178.75 KB/183040, currentSize=120.37 KB/123255 for 2444c4148e3a1c831bc6f992044b0533 in 880ms, sequenceid=98, compaction requested=true 2024-12-03T04:43:17,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:17,015 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:17,016 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:17,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:17,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:17,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:17,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:17,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:17,023 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-03T04:43:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:43:17,032 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:17,036 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:17,038 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:17,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:17,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=127.08 KB heapSize=367.55 KB 2024-12-03T04:43:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:17,057 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:17,064 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/B is initiating minor compaction (all files) 2024-12-03T04:43:17,065 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/B in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,066 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/840029dc1a774ce4bd69de492f93cfdd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/b451529e47bb46f1b2e7b452dd5a8be7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/569f45e390c94bbd8900a3b207df3b95, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88ea2bec57184cac8f58d6bd5d04f11a] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=46.9 K 2024-12-03T04:43:17,075 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 840029dc1a774ce4bd69de492f93cfdd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733200992682 2024-12-03T04:43:17,090 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting b451529e47bb46f1b2e7b452dd5a8be7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733200993031 2024-12-03T04:43:17,095 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 569f45e390c94bbd8900a3b207df3b95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733200993789 2024-12-03T04:43:17,098 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 88ea2bec57184cac8f58d6bd5d04f11a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733200995437 2024-12-03T04:43:17,101 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:17,102 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/A is initiating minor compaction (all files) 2024-12-03T04:43:17,102 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/A in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,102 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/e828cac9104947cfad9853660faee996, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a786a70b41a24a90b6c903e3ae30b67c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6bb91d2038da4f55832e964d463d743d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/d0c2819fd75844ab95d49a67ff1b7deb] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=46.9 K 2024-12-03T04:43:17,107 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting e828cac9104947cfad9853660faee996, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733200992682 2024-12-03T04:43:17,117 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting a786a70b41a24a90b6c903e3ae30b67c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733200993031 2024-12-03T04:43:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:43:17,137 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bb91d2038da4f55832e964d463d743d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733200993789 2024-12-03T04:43:17,139 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0c2819fd75844ab95d49a67ff1b7deb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733200995437 2024-12-03T04:43:17,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/56ca4a5b6a034b89bc59f48180be9304 is 50, key is test_row_0/A:col10/1733200997046/Put/seqid=0 2024-12-03T04:43:17,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741849_1025 (size=16781) 2024-12-03T04:43:17,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.15 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/56ca4a5b6a034b89bc59f48180be9304 2024-12-03T04:43:17,191 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#A#compaction#13 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:17,192 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/c4045edf0d4e40cfa88f6c0c79a1092b is 50, key is test_row_0/A:col10/1733200995437/Put/seqid=0 2024-12-03T04:43:17,201 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#B#compaction#14 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:17,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201057172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,203 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/6696b81cffdd4aee8e31e8659691b285 is 50, key is test_row_0/B:col10/1733200995437/Put/seqid=0 2024-12-03T04:43:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:17,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201057179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:17,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741850_1026 (size=12139) 2024-12-03T04:43:17,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/83af2fd015a044cfb2232148e6dba28e is 50, key is test_row_0/B:col10/1733200997046/Put/seqid=0 2024-12-03T04:43:17,259 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/c4045edf0d4e40cfa88f6c0c79a1092b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/c4045edf0d4e40cfa88f6c0c79a1092b 2024-12-03T04:43:17,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741851_1027 (size=12139) 2024-12-03T04:43:17,284 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/6696b81cffdd4aee8e31e8659691b285 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/6696b81cffdd4aee8e31e8659691b285 2024-12-03T04:43:17,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741852_1028 (size=12101) 2024-12-03T04:43:17,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/83af2fd015a044cfb2232148e6dba28e 2024-12-03T04:43:17,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201057312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:17,322 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/A of 2444c4148e3a1c831bc6f992044b0533 into c4045edf0d4e40cfa88f6c0c79a1092b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:17,322 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:17,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,323 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/B of 2444c4148e3a1c831bc6f992044b0533 into 6696b81cffdd4aee8e31e8659691b285(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201057313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,323 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:17,328 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/A, priority=12, startTime=1733200997017; duration=0sec 2024-12-03T04:43:17,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/cbb4c91d24de40418f01cbb14940baaa is 50, key is test_row_0/C:col10/1733200997046/Put/seqid=0 2024-12-03T04:43:17,328 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:17,329 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:A 2024-12-03T04:43:17,329 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:17,334 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/B, priority=12, startTime=1733200997020; duration=0sec 2024-12-03T04:43:17,335 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:17,335 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:B 2024-12-03T04:43:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:43:17,369 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:17,370 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/C is initiating minor compaction (all files) 2024-12-03T04:43:17,370 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/C in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,370 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a129253573a44b3eababd773271be6f9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f755edb8af9b411db220f4cd96b4fb17, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/435f7f0032af4c4697f5a932f0ccba6c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/94c2922d75204c36a9f0bc090791b5ed] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=46.9 K 2024-12-03T04:43:17,371 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting a129253573a44b3eababd773271be6f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733200992682 2024-12-03T04:43:17,372 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f755edb8af9b411db220f4cd96b4fb17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733200993031 2024-12-03T04:43:17,373 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 435f7f0032af4c4697f5a932f0ccba6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733200993789 2024-12-03T04:43:17,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:17,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,375 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94c2922d75204c36a9f0bc090791b5ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733200995437 2024-12-03T04:43:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741853_1029 (size=12101) 2024-12-03T04:43:17,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/cbb4c91d24de40418f01cbb14940baaa 2024-12-03T04:43:17,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/56ca4a5b6a034b89bc59f48180be9304 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/56ca4a5b6a034b89bc59f48180be9304 2024-12-03T04:43:17,416 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#C#compaction#17 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:17,417 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/70814b14c2674d83a525fbeae4181907 is 50, key is test_row_0/C:col10/1733200995437/Put/seqid=0 2024-12-03T04:43:17,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741854_1030 (size=12139) 2024-12-03T04:43:17,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/56ca4a5b6a034b89bc59f48180be9304, entries=250, sequenceid=130, filesize=16.4 K 2024-12-03T04:43:17,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/83af2fd015a044cfb2232148e6dba28e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/83af2fd015a044cfb2232148e6dba28e 2024-12-03T04:43:17,463 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/70814b14c2674d83a525fbeae4181907 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/70814b14c2674d83a525fbeae4181907 2024-12-03T04:43:17,475 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/C of 2444c4148e3a1c831bc6f992044b0533 into 70814b14c2674d83a525fbeae4181907(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:17,476 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:17,476 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/C, priority=12, startTime=1733200997020; duration=0sec 2024-12-03T04:43:17,476 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:17,476 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:C 2024-12-03T04:43:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201057472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:17,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201057472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:17,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/83af2fd015a044cfb2232148e6dba28e, entries=150, sequenceid=130, filesize=11.8 K 2024-12-03T04:43:17,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/cbb4c91d24de40418f01cbb14940baaa as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cbb4c91d24de40418f01cbb14940baaa 2024-12-03T04:43:17,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cbb4c91d24de40418f01cbb14940baaa, entries=150, sequenceid=130, filesize=11.8 K 2024-12-03T04:43:17,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~387.43 KB/396728, currentSize=46.96 KB/48090 for 2444c4148e3a1c831bc6f992044b0533 in 455ms, sequenceid=130, compaction requested=false 2024-12-03T04:43:17,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:17,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T04:43:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:17,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:17,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:17,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/6d74149b1813457e934d808d517c3ca8 is 50, key is test_row_0/A:col10/1733200997498/Put/seqid=0 2024-12-03T04:43:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741855_1031 (size=12151) 2024-12-03T04:43:17,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/6d74149b1813457e934d808d517c3ca8 2024-12-03T04:43:17,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/114e9c156c3648a3a9ebca3118a1e1a5 is 50, key is test_row_0/B:col10/1733200997498/Put/seqid=0 2024-12-03T04:43:17,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741856_1032 (size=12151) 2024-12-03T04:43:17,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/114e9c156c3648a3a9ebca3118a1e1a5 2024-12-03T04:43:17,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/c5968bdd3b4548fcacb6651a35b894b2 is 50, key is test_row_0/C:col10/1733200997498/Put/seqid=0 2024-12-03T04:43:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:43:17,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733200997599, max timestamp=1733200997655Num uniques 859; 2024-12-03T04:43:17,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733200997599, max timestamp=1733200997655Num uniques 859; 2024-12-03T04:43:17,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:43:17,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:43:17,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733200997599, max timestamp=1733200997655Num uniques 859; 2024-12-03T04:43:17,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:43:17,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:17,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741857_1033 (size=12151) 2024-12-03T04:43:17,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:17,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201057827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201057834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201057834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:17,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:17,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:17,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:17,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:17,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201057972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201057972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201057972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:18,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:18,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/c5968bdd3b4548fcacb6651a35b894b2 2024-12-03T04:43:18,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/6d74149b1813457e934d808d517c3ca8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6d74149b1813457e934d808d517c3ca8 2024-12-03T04:43:18,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6d74149b1813457e934d808d517c3ca8, entries=150, sequenceid=144, filesize=11.9 K 2024-12-03T04:43:18,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/114e9c156c3648a3a9ebca3118a1e1a5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/114e9c156c3648a3a9ebca3118a1e1a5 2024-12-03T04:43:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:43:18,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/114e9c156c3648a3a9ebca3118a1e1a5, entries=150, sequenceid=144, filesize=11.9 K 2024-12-03T04:43:18,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/c5968bdd3b4548fcacb6651a35b894b2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c5968bdd3b4548fcacb6651a35b894b2 2024-12-03T04:43:18,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c5968bdd3b4548fcacb6651a35b894b2, entries=150, sequenceid=144, filesize=11.9 K 2024-12-03T04:43:18,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=147.60 KB/151140 for 2444c4148e3a1c831bc6f992044b0533 in 683ms, sequenceid=144, compaction requested=true 2024-12-03T04:43:18,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:18,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:18,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:18,187 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:18,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:18,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:18,187 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:18,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:18,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:18,189 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41071 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:18,189 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/A is initiating minor compaction (all files) 2024-12-03T04:43:18,189 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/A in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,189 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/c4045edf0d4e40cfa88f6c0c79a1092b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/56ca4a5b6a034b89bc59f48180be9304, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6d74149b1813457e934d808d517c3ca8] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=40.1 K 2024-12-03T04:43:18,190 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:18,190 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/B is initiating minor compaction (all files) 2024-12-03T04:43:18,190 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/B in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,190 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/6696b81cffdd4aee8e31e8659691b285, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/83af2fd015a044cfb2232148e6dba28e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/114e9c156c3648a3a9ebca3118a1e1a5] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=35.5 K 2024-12-03T04:43:18,192 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4045edf0d4e40cfa88f6c0c79a1092b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733200995437 2024-12-03T04:43:18,192 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 6696b81cffdd4aee8e31e8659691b285, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733200995437 2024-12-03T04:43:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:18,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-03T04:43:18,193 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56ca4a5b6a034b89bc59f48180be9304, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733200996333 2024-12-03T04:43:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:18,193 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 83af2fd015a044cfb2232148e6dba28e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733200996345 2024-12-03T04:43:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:18,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:18,195 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 114e9c156c3648a3a9ebca3118a1e1a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733200997091 2024-12-03T04:43:18,195 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d74149b1813457e934d808d517c3ca8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733200997091 2024-12-03T04:43:18,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:18,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:18,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/bfadbfb15a1d4a649b7a1df4119ac7eb is 50, key is test_row_0/A:col10/1733200997814/Put/seqid=0 2024-12-03T04:43:18,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201058210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201058223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201058223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,232 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:18,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,233 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/8e66a1748448474ab325a808ad4e70af is 50, key is test_row_0/A:col10/1733200997498/Put/seqid=0 2024-12-03T04:43:18,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,260 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#B#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:18,262 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/a1c658101b9c40bfbf6e1f69446bfe1f is 50, key is test_row_0/B:col10/1733200997498/Put/seqid=0 2024-12-03T04:43:18,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741859_1035 (size=12391) 2024-12-03T04:43:18,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741858_1034 (size=14541) 2024-12-03T04:43:18,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/bfadbfb15a1d4a649b7a1df4119ac7eb 2024-12-03T04:43:18,329 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/8e66a1748448474ab325a808ad4e70af as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/8e66a1748448474ab325a808ad4e70af 2024-12-03T04:43:18,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201058338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201058338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201058342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,351 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/A of 2444c4148e3a1c831bc6f992044b0533 into 8e66a1748448474ab325a808ad4e70af(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:18,351 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:18,351 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/A, priority=13, startTime=1733200998187; duration=0sec 2024-12-03T04:43:18,352 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:18,352 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:A 2024-12-03T04:43:18,352 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,354 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:18,354 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/C is initiating minor compaction (all files) 2024-12-03T04:43:18,354 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/C in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,354 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/70814b14c2674d83a525fbeae4181907, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cbb4c91d24de40418f01cbb14940baaa, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c5968bdd3b4548fcacb6651a35b894b2] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=35.5 K 2024-12-03T04:43:18,357 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70814b14c2674d83a525fbeae4181907, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733200995437 2024-12-03T04:43:18,358 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbb4c91d24de40418f01cbb14940baaa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733200996345 2024-12-03T04:43:18,358 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5968bdd3b4548fcacb6651a35b894b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733200997091 2024-12-03T04:43:18,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:18,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:18,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741860_1036 (size=12391) 2024-12-03T04:43:18,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/f994d5ba872244b1a6f7a7ba5ad6d0c2 is 50, key is test_row_0/B:col10/1733200997814/Put/seqid=0 2024-12-03T04:43:18,378 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/a1c658101b9c40bfbf6e1f69446bfe1f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/a1c658101b9c40bfbf6e1f69446bfe1f 2024-12-03T04:43:18,394 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#C#compaction#25 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:18,395 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/334fdc3de91342abbd939dd5aad18d24 is 50, key is test_row_0/C:col10/1733200997498/Put/seqid=0 2024-12-03T04:43:18,406 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/B of 2444c4148e3a1c831bc6f992044b0533 into a1c658101b9c40bfbf6e1f69446bfe1f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:18,406 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:18,406 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/B, priority=13, startTime=1733200998187; duration=0sec 2024-12-03T04:43:18,406 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:18,407 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:B 2024-12-03T04:43:18,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741861_1037 (size=12151) 2024-12-03T04:43:18,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741862_1038 (size=12391) 2024-12-03T04:43:18,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201058494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201058494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,516 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 2444c4148e3a1c831bc6f992044b0533 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:18,527 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 2444c4148e3a1c831bc6f992044b0533 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:18,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201058554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201058562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201058565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:18,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:18,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:18,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:18,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:18,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/f994d5ba872244b1a6f7a7ba5ad6d0c2 2024-12-03T04:43:18,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201058878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,892 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/334fdc3de91342abbd939dd5aad18d24 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/334fdc3de91342abbd939dd5aad18d24 2024-12-03T04:43:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/ea9baf092d07489ea5cd92abeb4a3b2b is 50, key is test_row_0/C:col10/1733200997814/Put/seqid=0 2024-12-03T04:43:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201058879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201058881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:18,911 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/C of 2444c4148e3a1c831bc6f992044b0533 into 334fdc3de91342abbd939dd5aad18d24(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:18,911 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:18,911 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/C, priority=13, startTime=1733200998187; duration=0sec 2024-12-03T04:43:18,911 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:18,911 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:C 2024-12-03T04:43:18,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741863_1039 (size=12151) 2024-12-03T04:43:18,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/ea9baf092d07489ea5cd92abeb4a3b2b 2024-12-03T04:43:18,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/bfadbfb15a1d4a649b7a1df4119ac7eb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/bfadbfb15a1d4a649b7a1df4119ac7eb 2024-12-03T04:43:18,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/bfadbfb15a1d4a649b7a1df4119ac7eb, entries=200, sequenceid=181, filesize=14.2 K 2024-12-03T04:43:18,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/f994d5ba872244b1a6f7a7ba5ad6d0c2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/f994d5ba872244b1a6f7a7ba5ad6d0c2 2024-12-03T04:43:18,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/f994d5ba872244b1a6f7a7ba5ad6d0c2, entries=150, sequenceid=181, filesize=11.9 K 2024-12-03T04:43:19,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/ea9baf092d07489ea5cd92abeb4a3b2b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ea9baf092d07489ea5cd92abeb4a3b2b 2024-12-03T04:43:19,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:19,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:19,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:19,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:19,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:19,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:19,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ea9baf092d07489ea5cd92abeb4a3b2b, entries=150, sequenceid=181, filesize=11.9 K 2024-12-03T04:43:19,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 2444c4148e3a1c831bc6f992044b0533 in 827ms, sequenceid=181, compaction requested=false 2024-12-03T04:43:19,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:19,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T04:43:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:19,158 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T04:43:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:19,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:43:19,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/dae87fc79c54453b8fa1c8275af765cd is 50, key is test_row_0/A:col10/1733200998219/Put/seqid=0 2024-12-03T04:43:19,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741864_1040 (size=12151) 2024-12-03T04:43:19,220 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/dae87fc79c54453b8fa1c8275af765cd 2024-12-03T04:43:19,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/88c30c8ff7b143e4a543f599e34d1505 is 50, key is test_row_0/B:col10/1733200998219/Put/seqid=0 2024-12-03T04:43:19,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741865_1041 (size=12151) 2024-12-03T04:43:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:19,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:19,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733200999467, max timestamp=1733200999523Num uniques 859; 2024-12-03T04:43:19,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:43:19,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733200999467, max timestamp=1733200999523Num uniques 859; 2024-12-03T04:43:19,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:43:19,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733200999467, max timestamp=1733200999523Num uniques 859; 2024-12-03T04:43:19,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:43:19,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:19,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733200999601, max timestamp=1733200999637Num uniques 1168; 2024-12-03T04:43:19,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-12-03T04:43:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201059630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:19,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733200999601, max timestamp=1733200999637Num uniques 1168; 2024-12-03T04:43:19,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-12-03T04:43:19,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733200999601, max timestamp=1733200999637Num uniques 1168; 2024-12-03T04:43:19,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-12-03T04:43:19,713 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/88c30c8ff7b143e4a543f599e34d1505 2024-12-03T04:43:19,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/cfd55afcd7134532b2e7d0000fbac48b is 50, key is test_row_0/C:col10/1733200998219/Put/seqid=0 2024-12-03T04:43:19,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741866_1042 (size=12151) 2024-12-03T04:43:19,812 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/cfd55afcd7134532b2e7d0000fbac48b 2024-12-03T04:43:19,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201059796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:19,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:19,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/dae87fc79c54453b8fa1c8275af765cd as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/dae87fc79c54453b8fa1c8275af765cd 2024-12-03T04:43:19,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201059811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201059815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:19,833 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/dae87fc79c54453b8fa1c8275af765cd, entries=150, sequenceid=192, filesize=11.9 K 2024-12-03T04:43:19,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/88c30c8ff7b143e4a543f599e34d1505 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88c30c8ff7b143e4a543f599e34d1505 2024-12-03T04:43:19,854 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88c30c8ff7b143e4a543f599e34d1505, entries=150, sequenceid=192, filesize=11.9 K 2024-12-03T04:43:19,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/cfd55afcd7134532b2e7d0000fbac48b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cfd55afcd7134532b2e7d0000fbac48b 2024-12-03T04:43:19,874 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cfd55afcd7134532b2e7d0000fbac48b, entries=150, sequenceid=192, filesize=11.9 K 2024-12-03T04:43:19,876 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 2444c4148e3a1c831bc6f992044b0533 in 718ms, sequenceid=192, compaction requested=true 2024-12-03T04:43:19,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:19,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:19,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-03T04:43:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-03T04:43:19,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T04:43:19,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8450 sec 2024-12-03T04:43:19,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.8730 sec 2024-12-03T04:43:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:19,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=161.02 KB heapSize=450.47 KB 2024-12-03T04:43:19,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:19,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:43:19,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:19,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:43:19,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:19,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:43:19,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/a1ee1c0c97b64c52b1ea498d84d1f6df is 50, key is test_row_0/A:col10/1733200999931/Put/seqid=0 2024-12-03T04:43:19,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201059954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:19,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201059956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:19,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201059969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741867_1043 (size=12151) 2024-12-03T04:43:20,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/a1ee1c0c97b64c52b1ea498d84d1f6df 2024-12-03T04:43:20,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/89b6d62d86684a299393a833d712dd36 is 50, key is test_row_0/B:col10/1733200999931/Put/seqid=0 2024-12-03T04:43:20,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201060082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201060082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201060083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741868_1044 (size=12151) 2024-12-03T04:43:20,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/89b6d62d86684a299393a833d712dd36 2024-12-03T04:43:20,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/48f3e99a1c79453796baa8684b0918b8 is 50, key is test_row_0/C:col10/1733200999931/Put/seqid=0 2024-12-03T04:43:20,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741869_1045 (size=12151) 2024-12-03T04:43:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201060296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201060295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201060302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201060546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:20,548 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 2444c4148e3a1c831bc6f992044b0533 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201060546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:20,564 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 2444c4148e3a1c831bc6f992044b0533 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/48f3e99a1c79453796baa8684b0918b8 2024-12-03T04:43:20,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201060613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:20,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:20,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201060613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:20,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201060613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:20,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/a1ee1c0c97b64c52b1ea498d84d1f6df as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a1ee1c0c97b64c52b1ea498d84d1f6df 2024-12-03T04:43:20,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a1ee1c0c97b64c52b1ea498d84d1f6df, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T04:43:20,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/89b6d62d86684a299393a833d712dd36 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/89b6d62d86684a299393a833d712dd36 2024-12-03T04:43:20,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/89b6d62d86684a299393a833d712dd36, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T04:43:20,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/48f3e99a1c79453796baa8684b0918b8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/48f3e99a1c79453796baa8684b0918b8 2024-12-03T04:43:20,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/48f3e99a1c79453796baa8684b0918b8, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T04:43:20,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~470.34 KB/481632, currentSize=20.13 KB/20610 for 2444c4148e3a1c831bc6f992044b0533 in 747ms, sequenceid=248, compaction requested=true 2024-12-03T04:43:20,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:20,693 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:20,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:20,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:20,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:20,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:20,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:20,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T04:43:20,693 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:20,696 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51234 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:20,696 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:20,697 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/B is initiating minor compaction (all files) 2024-12-03T04:43:20,697 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/A is initiating minor compaction (all files) 2024-12-03T04:43:20,697 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/B in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:20,697 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/A in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:20,697 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/a1c658101b9c40bfbf6e1f69446bfe1f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/f994d5ba872244b1a6f7a7ba5ad6d0c2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88c30c8ff7b143e4a543f599e34d1505, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/89b6d62d86684a299393a833d712dd36] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=47.7 K 2024-12-03T04:43:20,697 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/8e66a1748448474ab325a808ad4e70af, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/bfadbfb15a1d4a649b7a1df4119ac7eb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/dae87fc79c54453b8fa1c8275af765cd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a1ee1c0c97b64c52b1ea498d84d1f6df] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=50.0 K 2024-12-03T04:43:20,698 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a1c658101b9c40bfbf6e1f69446bfe1f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733200997091 2024-12-03T04:43:20,698 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e66a1748448474ab325a808ad4e70af, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733200997091 2024-12-03T04:43:20,699 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfadbfb15a1d4a649b7a1df4119ac7eb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733200997812 2024-12-03T04:43:20,699 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting f994d5ba872244b1a6f7a7ba5ad6d0c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733200997814 2024-12-03T04:43:20,700 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting dae87fc79c54453b8fa1c8275af765cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733200998206 2024-12-03T04:43:20,700 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 88c30c8ff7b143e4a543f599e34d1505, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733200998206 2024-12-03T04:43:20,701 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1ee1c0c97b64c52b1ea498d84d1f6df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733200999763 2024-12-03T04:43:20,701 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 89b6d62d86684a299393a833d712dd36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733200999763 2024-12-03T04:43:20,749 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#B#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:20,750 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/978f6de831af4027bb275d73dfde2908 is 50, key is test_row_0/B:col10/1733200999931/Put/seqid=0 2024-12-03T04:43:20,760 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:20,761 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/fdc9526507ed410f984b5919aad7a287 is 50, key is test_row_0/A:col10/1733200999931/Put/seqid=0 2024-12-03T04:43:20,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741870_1046 (size=12527) 2024-12-03T04:43:20,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741871_1047 (size=12527) 2024-12-03T04:43:20,817 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/978f6de831af4027bb275d73dfde2908 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/978f6de831af4027bb275d73dfde2908 2024-12-03T04:43:20,837 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/B of 2444c4148e3a1c831bc6f992044b0533 into 978f6de831af4027bb275d73dfde2908(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:20,837 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:20,838 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/B, priority=12, startTime=1733201000693; duration=0sec 2024-12-03T04:43:20,838 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:20,838 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:B 2024-12-03T04:43:20,839 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:20,848 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:20,848 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/C is initiating minor compaction (all files) 2024-12-03T04:43:20,848 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/C in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:20,848 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/334fdc3de91342abbd939dd5aad18d24, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ea9baf092d07489ea5cd92abeb4a3b2b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cfd55afcd7134532b2e7d0000fbac48b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/48f3e99a1c79453796baa8684b0918b8] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=47.7 K 2024-12-03T04:43:20,850 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 334fdc3de91342abbd939dd5aad18d24, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733200997091 2024-12-03T04:43:20,851 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ea9baf092d07489ea5cd92abeb4a3b2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733200997814 2024-12-03T04:43:20,851 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting cfd55afcd7134532b2e7d0000fbac48b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733200998206 2024-12-03T04:43:20,852 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 48f3e99a1c79453796baa8684b0918b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733200999763 2024-12-03T04:43:20,899 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#C#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:20,900 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/221a0982816641b3931553958a2246fe is 50, key is test_row_0/C:col10/1733200999931/Put/seqid=0 2024-12-03T04:43:20,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741872_1048 (size=12527) 2024-12-03T04:43:20,938 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/221a0982816641b3931553958a2246fe as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/221a0982816641b3931553958a2246fe 2024-12-03T04:43:20,966 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/C of 2444c4148e3a1c831bc6f992044b0533 into 221a0982816641b3931553958a2246fe(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:20,966 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:20,967 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/C, priority=12, startTime=1733201000693; duration=0sec 2024-12-03T04:43:20,967 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:20,967 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:C 2024-12-03T04:43:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:21,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:43:21,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:21,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:21,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:21,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:21,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:21,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:21,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/9fbe1425bf75498fb3b398f92d32c80e is 50, key is test_row_0/A:col10/1733201001135/Put/seqid=0 2024-12-03T04:43:21,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741873_1049 (size=12251) 2024-12-03T04:43:21,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/9fbe1425bf75498fb3b398f92d32c80e 2024-12-03T04:43:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T04:43:21,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/af0e645873c243febda1e21d42c5817d is 50, key is test_row_0/B:col10/1733201001135/Put/seqid=0 2024-12-03T04:43:21,227 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:21,229 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-03T04:43:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:43:21,242 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:21,244 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:21,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:21,263 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/fdc9526507ed410f984b5919aad7a287 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/fdc9526507ed410f984b5919aad7a287 2024-12-03T04:43:21,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741874_1050 (size=12251) 2024-12-03T04:43:21,276 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/A of 2444c4148e3a1c831bc6f992044b0533 into fdc9526507ed410f984b5919aad7a287(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:21,276 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:21,276 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/A, priority=12, startTime=1733201000693; duration=0sec 2024-12-03T04:43:21,276 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:21,276 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:A 2024-12-03T04:43:21,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201061291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:21,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201061298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201061307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:43:21,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T04:43:21,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:21,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:21,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:21,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201061417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201061421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201061422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:21,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T04:43:21,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:21,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:21,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:43:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201061634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201061637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201061641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:21,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/af0e645873c243febda1e21d42c5817d 2024-12-03T04:43:21,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/ca07d48c5107412a8d68317fff21e017 is 50, key is test_row_0/C:col10/1733201001135/Put/seqid=0 2024-12-03T04:43:21,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T04:43:21,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:21,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:21,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:21,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741875_1051 (size=12251) 2024-12-03T04:43:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:43:21,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T04:43:21,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:21,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:21,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:21,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:21,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201061952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:21,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201061962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:21,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201061965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:22,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T04:43:22,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:22,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:22,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:22,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:22,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:22,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/ca07d48c5107412a8d68317fff21e017 2024-12-03T04:43:22,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/9fbe1425bf75498fb3b398f92d32c80e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9fbe1425bf75498fb3b398f92d32c80e 2024-12-03T04:43:22,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9fbe1425bf75498fb3b398f92d32c80e, entries=150, sequenceid=262, filesize=12.0 K 2024-12-03T04:43:22,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/af0e645873c243febda1e21d42c5817d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/af0e645873c243febda1e21d42c5817d 2024-12-03T04:43:22,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/af0e645873c243febda1e21d42c5817d, entries=150, sequenceid=262, filesize=12.0 K 2024-12-03T04:43:22,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/ca07d48c5107412a8d68317fff21e017 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ca07d48c5107412a8d68317fff21e017 2024-12-03T04:43:22,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T04:43:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:22,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:22,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ca07d48c5107412a8d68317fff21e017, entries=150, sequenceid=262, filesize=12.0 K 2024-12-03T04:43:22,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2444c4148e3a1c831bc6f992044b0533 in 1077ms, sequenceid=262, compaction requested=false 2024-12-03T04:43:22,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:22,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T04:43:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:22,364 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:43:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:22,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/ab3e0e1b8abd46de8ecc01852d0af92c is 50, key is test_row_0/A:col10/1733201001297/Put/seqid=0 2024-12-03T04:43:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:43:22,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741876_1052 (size=12301) 2024-12-03T04:43:22,403 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/ab3e0e1b8abd46de8ecc01852d0af92c 2024-12-03T04:43:22,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/cfcc5dc9036b458a8cac3cc61a25d5af is 50, key is test_row_0/B:col10/1733201001297/Put/seqid=0 2024-12-03T04:43:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:22,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:22,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741877_1053 (size=12301) 2024-12-03T04:43:22,489 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/cfcc5dc9036b458a8cac3cc61a25d5af 2024-12-03T04:43:22,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/a20403c1d5ea4b68bf4f506b00ccdd4e is 50, key is test_row_0/C:col10/1733201001297/Put/seqid=0 2024-12-03T04:43:22,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201062502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:22,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201062503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:22,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201062506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:22,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741878_1054 (size=12301) 2024-12-03T04:43:22,571 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/a20403c1d5ea4b68bf4f506b00ccdd4e 2024-12-03T04:43:22,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/ab3e0e1b8abd46de8ecc01852d0af92c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/ab3e0e1b8abd46de8ecc01852d0af92c 2024-12-03T04:43:22,596 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/ab3e0e1b8abd46de8ecc01852d0af92c, entries=150, sequenceid=283, filesize=12.0 K 2024-12-03T04:43:22,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/cfcc5dc9036b458a8cac3cc61a25d5af as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/cfcc5dc9036b458a8cac3cc61a25d5af 2024-12-03T04:43:22,612 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/cfcc5dc9036b458a8cac3cc61a25d5af, entries=150, sequenceid=283, filesize=12.0 K 2024-12-03T04:43:22,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/a20403c1d5ea4b68bf4f506b00ccdd4e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a20403c1d5ea4b68bf4f506b00ccdd4e 2024-12-03T04:43:22,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201062622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:22,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201062622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201062622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:22,641 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a20403c1d5ea4b68bf4f506b00ccdd4e, entries=150, sequenceid=283, filesize=12.0 K 2024-12-03T04:43:22,644 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2444c4148e3a1c831bc6f992044b0533 in 280ms, sequenceid=283, compaction requested=true 2024-12-03T04:43:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:22,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:22,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-03T04:43:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-03T04:43:22,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T04:43:22,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4070 sec 2024-12-03T04:43:22,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.4270 sec 2024-12-03T04:43:22,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T04:43:22,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:22,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:22,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:22,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:22,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:22,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:22,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/21b85f1ff2954daaafb553b5fcf23055 is 50, key is test_row_0/A:col10/1733201002504/Put/seqid=0 2024-12-03T04:43:22,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741879_1055 (size=12301) 2024-12-03T04:43:22,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/21b85f1ff2954daaafb553b5fcf23055 2024-12-03T04:43:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201062941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201062942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:22,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201062963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:22,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/03c47e83168c4deeb0c98c482dd9d7a0 is 50, key is test_row_0/B:col10/1733201002504/Put/seqid=0 2024-12-03T04:43:22,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741880_1056 (size=12301) 2024-12-03T04:43:22,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/03c47e83168c4deeb0c98c482dd9d7a0 2024-12-03T04:43:23,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/695564ad659043ff931ed3ea07e0c804 is 50, key is test_row_0/C:col10/1733201002504/Put/seqid=0 2024-12-03T04:43:23,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741881_1057 (size=12301) 2024-12-03T04:43:23,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/695564ad659043ff931ed3ea07e0c804 2024-12-03T04:43:23,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/21b85f1ff2954daaafb553b5fcf23055 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/21b85f1ff2954daaafb553b5fcf23055 2024-12-03T04:43:23,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201063072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/21b85f1ff2954daaafb553b5fcf23055, entries=150, sequenceid=296, filesize=12.0 K 2024-12-03T04:43:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201063072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/03c47e83168c4deeb0c98c482dd9d7a0 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/03c47e83168c4deeb0c98c482dd9d7a0 2024-12-03T04:43:23,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/03c47e83168c4deeb0c98c482dd9d7a0, entries=150, sequenceid=296, filesize=12.0 K 2024-12-03T04:43:23,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/695564ad659043ff931ed3ea07e0c804 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/695564ad659043ff931ed3ea07e0c804 2024-12-03T04:43:23,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201063082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/695564ad659043ff931ed3ea07e0c804, entries=150, sequenceid=296, filesize=12.0 K 2024-12-03T04:43:23,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2444c4148e3a1c831bc6f992044b0533 in 260ms, sequenceid=296, compaction requested=true 2024-12-03T04:43:23,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:23,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:23,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:23,111 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:23,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:23,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:23,111 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:23,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:23,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:23,113 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49380 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:23,113 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/A is initiating minor compaction (all files) 2024-12-03T04:43:23,113 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/A in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:23,113 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/fdc9526507ed410f984b5919aad7a287, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9fbe1425bf75498fb3b398f92d32c80e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/ab3e0e1b8abd46de8ecc01852d0af92c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/21b85f1ff2954daaafb553b5fcf23055] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=48.2 K 2024-12-03T04:43:23,115 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49380 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:23,115 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/B is initiating minor compaction (all files) 2024-12-03T04:43:23,115 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/B in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:23,115 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/978f6de831af4027bb275d73dfde2908, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/af0e645873c243febda1e21d42c5817d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/cfcc5dc9036b458a8cac3cc61a25d5af, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/03c47e83168c4deeb0c98c482dd9d7a0] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=48.2 K 2024-12-03T04:43:23,115 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc9526507ed410f984b5919aad7a287, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733200999763 2024-12-03T04:43:23,116 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 978f6de831af4027bb275d73dfde2908, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733200999763 2024-12-03T04:43:23,116 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fbe1425bf75498fb3b398f92d32c80e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733200999962 2024-12-03T04:43:23,116 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting af0e645873c243febda1e21d42c5817d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733200999962 2024-12-03T04:43:23,117 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab3e0e1b8abd46de8ecc01852d0af92c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733201001282 2024-12-03T04:43:23,117 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting cfcc5dc9036b458a8cac3cc61a25d5af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733201001282 2024-12-03T04:43:23,118 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21b85f1ff2954daaafb553b5fcf23055, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201002483 2024-12-03T04:43:23,118 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c47e83168c4deeb0c98c482dd9d7a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201002483 2024-12-03T04:43:23,147 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:23,148 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/30b4975d2a4c433b84b0d9600edf2383 is 50, key is test_row_0/A:col10/1733201002504/Put/seqid=0 2024-12-03T04:43:23,172 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#B#compaction#46 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:23,173 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/908ed7dc2d6848d6ba7032b3c278bc21 is 50, key is test_row_0/B:col10/1733201002504/Put/seqid=0 2024-12-03T04:43:23,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741882_1058 (size=12813) 2024-12-03T04:43:23,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741883_1059 (size=12813) 2024-12-03T04:43:23,265 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/908ed7dc2d6848d6ba7032b3c278bc21 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/908ed7dc2d6848d6ba7032b3c278bc21 2024-12-03T04:43:23,278 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/B of 2444c4148e3a1c831bc6f992044b0533 into 908ed7dc2d6848d6ba7032b3c278bc21(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:23,278 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:23,278 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/B, priority=12, startTime=1733201003111; duration=0sec 2024-12-03T04:43:23,278 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:23,278 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:B 2024-12-03T04:43:23,279 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:23,284 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49380 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:23,284 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/C is initiating minor compaction (all files) 2024-12-03T04:43:23,285 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/C in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:23,285 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/221a0982816641b3931553958a2246fe, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ca07d48c5107412a8d68317fff21e017, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a20403c1d5ea4b68bf4f506b00ccdd4e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/695564ad659043ff931ed3ea07e0c804] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=48.2 K 2024-12-03T04:43:23,291 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 221a0982816641b3931553958a2246fe, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733200999763 2024-12-03T04:43:23,291 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ca07d48c5107412a8d68317fff21e017, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733200999962 2024-12-03T04:43:23,293 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a20403c1d5ea4b68bf4f506b00ccdd4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733201001282 2024-12-03T04:43:23,293 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 695564ad659043ff931ed3ea07e0c804, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201002483 2024-12-03T04:43:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:23,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:43:23,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201002933, max timestamp=1733201003294Num uniques 859; 2024-12-03T04:43:23,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:23,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201002933, max timestamp=1733201003294Num uniques 859; 2024-12-03T04:43:23,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:23,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:23,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:23,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:23,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:23,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:23,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:23,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/8a8be42760de42b1856d7f1254d4d719 is 50, key is test_row_0/A:col10/1733201003294/Put/seqid=0 2024-12-03T04:43:23,332 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:23,333 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/ece095c0bccf4fa4b8e5a4c2169519f3 is 50, key is test_row_0/C:col10/1733201002504/Put/seqid=0 2024-12-03T04:43:23,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741884_1060 (size=14741) 2024-12-03T04:43:23,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/8a8be42760de42b1856d7f1254d4d719 2024-12-03T04:43:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T04:43:23,392 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:23,393 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-03T04:43:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:43:23,395 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:23,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:23,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:23,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741885_1061 (size=12813) 2024-12-03T04:43:23,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/28850eaf88034aadb094644607dcb607 is 50, key is test_row_0/B:col10/1733201003294/Put/seqid=0 2024-12-03T04:43:23,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733201003366, max timestamp=1733201003414Num uniques 859; 2024-12-03T04:43:23,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:43:23,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733201003366, max timestamp=1733201003414Num uniques 859; 2024-12-03T04:43:23,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733201003366, max timestamp=1733201003414Num uniques 859; 2024-12-03T04:43:23,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:43:23,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:43:23,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741886_1062 (size=12301) 2024-12-03T04:43:23,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/28850eaf88034aadb094644607dcb607 2024-12-03T04:43:23,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/54ba73547c194c00bf50c8bd6b23fa3e is 50, key is test_row_0/C:col10/1733201003294/Put/seqid=0 2024-12-03T04:43:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:43:23,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201063509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741887_1063 (size=12301) 2024-12-03T04:43:23,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/54ba73547c194c00bf50c8bd6b23fa3e 2024-12-03T04:43:23,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201063516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201063519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/8a8be42760de42b1856d7f1254d4d719 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/8a8be42760de42b1856d7f1254d4d719 2024-12-03T04:43:23,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:23,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:23,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:23,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:23,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:23,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:23,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/8a8be42760de42b1856d7f1254d4d719, entries=200, sequenceid=318, filesize=14.4 K 2024-12-03T04:43:23,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/28850eaf88034aadb094644607dcb607 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/28850eaf88034aadb094644607dcb607 2024-12-03T04:43:23,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/28850eaf88034aadb094644607dcb607, entries=150, sequenceid=318, filesize=12.0 K 2024-12-03T04:43:23,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/54ba73547c194c00bf50c8bd6b23fa3e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/54ba73547c194c00bf50c8bd6b23fa3e 2024-12-03T04:43:23,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/54ba73547c194c00bf50c8bd6b23fa3e, entries=150, sequenceid=318, filesize=12.0 K 2024-12-03T04:43:23,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~169.20 KB/173264, currentSize=127.47 KB/130530 for 2444c4148e3a1c831bc6f992044b0533 in 316ms, sequenceid=318, compaction requested=false 2024-12-03T04:43:23,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:23,645 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/30b4975d2a4c433b84b0d9600edf2383 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/30b4975d2a4c433b84b0d9600edf2383 2024-12-03T04:43:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:23,659 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/A of 2444c4148e3a1c831bc6f992044b0533 into 30b4975d2a4c433b84b0d9600edf2383(size=12.5 K), total size for store is 26.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:23,659 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:23,659 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/A, priority=12, startTime=1733201003111; duration=0sec 2024-12-03T04:43:23,659 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:23,659 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:A 2024-12-03T04:43:23,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-03T04:43:23,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:23,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:23,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:23,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:23,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:23,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:23,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/2f267023e32b4d0b95da9c4653750f93 is 50, key is test_row_0/A:col10/1733201003659/Put/seqid=0 2024-12-03T04:43:23,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201063691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201063697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201063700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:43:23,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:23,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:23,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:23,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:23,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:23,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:23,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741888_1064 (size=14741) 2024-12-03T04:43:23,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201063812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201063812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,828 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/ece095c0bccf4fa4b8e5a4c2169519f3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ece095c0bccf4fa4b8e5a4c2169519f3 2024-12-03T04:43:23,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201063822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:23,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:23,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:23,853 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/C of 2444c4148e3a1c831bc6f992044b0533 into ece095c0bccf4fa4b8e5a4c2169519f3(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:23,853 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:23,853 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/C, priority=12, startTime=1733201003111; duration=0sec 2024-12-03T04:43:23,853 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:23,853 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:C 2024-12-03T04:43:23,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:23,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:43:24,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201064031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:24,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201064032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:24,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:24,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:24,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201064043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:24,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/2f267023e32b4d0b95da9c4653750f93 2024-12-03T04:43:24,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/7df98600750e4a59bdc0214b916f1757 is 50, key is test_row_0/B:col10/1733201003659/Put/seqid=0 2024-12-03T04:43:24,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:24,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:24,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741889_1065 (size=12301) 2024-12-03T04:43:24,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/7df98600750e4a59bdc0214b916f1757 2024-12-03T04:43:24,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/bdfe73bcf62b4b0ba2ac6582c134bd95 is 50, key is test_row_0/C:col10/1733201003659/Put/seqid=0 2024-12-03T04:43:24,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201064351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:24,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201064352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:24,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:24,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741890_1066 (size=12301) 2024-12-03T04:43:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:24,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201064367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:24,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:24,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:24,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:43:24,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:24,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:24,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46534 deadline: 1733201064592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:24,598 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 2444c4148e3a1c831bc6f992044b0533 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8329 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:24,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46486 deadline: 1733201064601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:24,609 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 2444c4148e3a1c831bc6f992044b0533 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8332 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:24,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:24,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:24,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/bdfe73bcf62b4b0ba2ac6582c134bd95 2024-12-03T04:43:24,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/2f267023e32b4d0b95da9c4653750f93 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/2f267023e32b4d0b95da9c4653750f93 2024-12-03T04:43:24,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/2f267023e32b4d0b95da9c4653750f93, entries=200, sequenceid=355, filesize=14.4 K 2024-12-03T04:43:24,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/7df98600750e4a59bdc0214b916f1757 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/7df98600750e4a59bdc0214b916f1757 2024-12-03T04:43:24,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/7df98600750e4a59bdc0214b916f1757, entries=150, sequenceid=355, filesize=12.0 K 2024-12-03T04:43:24,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/bdfe73bcf62b4b0ba2ac6582c134bd95 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/bdfe73bcf62b4b0ba2ac6582c134bd95 2024-12-03T04:43:24,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/bdfe73bcf62b4b0ba2ac6582c134bd95, entries=150, sequenceid=355, filesize=12.0 K 2024-12-03T04:43:24,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 2444c4148e3a1c831bc6f992044b0533 in 1183ms, sequenceid=355, compaction requested=true 2024-12-03T04:43:24,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:24,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:24,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:24,844 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:24,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:24,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:24,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:24,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T04:43:24,844 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:24,846 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42295 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:24,846 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/A is initiating minor compaction (all files) 2024-12-03T04:43:24,846 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/A in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,847 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/30b4975d2a4c433b84b0d9600edf2383, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/8a8be42760de42b1856d7f1254d4d719, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/2f267023e32b4d0b95da9c4653750f93] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=41.3 K 2024-12-03T04:43:24,847 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:24,848 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/B is initiating minor compaction (all files) 2024-12-03T04:43:24,848 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/B in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,848 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/908ed7dc2d6848d6ba7032b3c278bc21, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/28850eaf88034aadb094644607dcb607, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/7df98600750e4a59bdc0214b916f1757] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=36.5 K 2024-12-03T04:43:24,848 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30b4975d2a4c433b84b0d9600edf2383, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201002483 2024-12-03T04:43:24,849 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 908ed7dc2d6848d6ba7032b3c278bc21, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201002483 2024-12-03T04:43:24,849 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a8be42760de42b1856d7f1254d4d719, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733201002933 2024-12-03T04:43:24,850 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 28850eaf88034aadb094644607dcb607, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733201002933 2024-12-03T04:43:24,850 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f267023e32b4d0b95da9c4653750f93, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733201003492 2024-12-03T04:43:24,851 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 7df98600750e4a59bdc0214b916f1757, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733201003651 2024-12-03T04:43:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:24,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:43:24,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:24,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:24,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:24,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:24,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:24,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:24,880 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#A#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:24,880 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/b4cd6d41b46c48d6992ce840c889c176 is 50, key is test_row_0/A:col10/1733201003659/Put/seqid=0 2024-12-03T04:43:24,888 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:24,889 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/083c0bba7df043f8a2df21cd9ec64279 is 50, key is test_row_0/B:col10/1733201003659/Put/seqid=0 2024-12-03T04:43:24,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:24,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/478a0af6c6b94920960d4f0b1b73b8ca is 50, key is test_row_0/A:col10/1733201003695/Put/seqid=0 2024-12-03T04:43:24,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741892_1068 (size=12915) 2024-12-03T04:43:24,970 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/083c0bba7df043f8a2df21cd9ec64279 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/083c0bba7df043f8a2df21cd9ec64279 2024-12-03T04:43:24,979 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/B of 2444c4148e3a1c831bc6f992044b0533 into 083c0bba7df043f8a2df21cd9ec64279(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:24,981 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:24,981 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/B, priority=13, startTime=1733201004844; duration=0sec 2024-12-03T04:43:24,981 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:24,981 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:B 2024-12-03T04:43:24,981 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:24,984 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:24,984 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/C is initiating minor compaction (all files) 2024-12-03T04:43:24,984 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/C in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:24,984 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ece095c0bccf4fa4b8e5a4c2169519f3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/54ba73547c194c00bf50c8bd6b23fa3e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/bdfe73bcf62b4b0ba2ac6582c134bd95] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=36.5 K 2024-12-03T04:43:24,987 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ece095c0bccf4fa4b8e5a4c2169519f3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733201002483 2024-12-03T04:43:24,987 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ba73547c194c00bf50c8bd6b23fa3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733201002933 2024-12-03T04:43:24,988 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting bdfe73bcf62b4b0ba2ac6582c134bd95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733201003651 2024-12-03T04:43:24,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741891_1067 (size=12915) 2024-12-03T04:43:25,002 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/b4cd6d41b46c48d6992ce840c889c176 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/b4cd6d41b46c48d6992ce840c889c176 2024-12-03T04:43:25,012 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/A of 2444c4148e3a1c831bc6f992044b0533 into b4cd6d41b46c48d6992ce840c889c176(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:25,012 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:25,012 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/A, priority=13, startTime=1733201004844; duration=0sec 2024-12-03T04:43:25,013 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:25,013 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:A 2024-12-03T04:43:25,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741893_1069 (size=14741) 2024-12-03T04:43:25,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/478a0af6c6b94920960d4f0b1b73b8ca 2024-12-03T04:43:25,040 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:25,041 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/ee63d87def49436aa3da1e792816c8a8 is 50, key is test_row_0/C:col10/1733201003659/Put/seqid=0 2024-12-03T04:43:25,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:25,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:25,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:25,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:25,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1733201004928, max timestamp=1733201005037Num uniques 859; 2024-12-03T04:43:25,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-12-03T04:43:25,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1733201004928, max timestamp=1733201005037Num uniques 859; 2024-12-03T04:43:25,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-12-03T04:43:25,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201004965, max timestamp=1733201005037Num uniques 859; 2024-12-03T04:43:25,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:25,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/e48c3fac3d544d0d85823a42655cc1f7 is 50, key is test_row_0/B:col10/1733201003695/Put/seqid=0 2024-12-03T04:43:25,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741894_1070 (size=12915) 2024-12-03T04:43:25,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741895_1071 (size=12301) 2024-12-03T04:43:25,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201065156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:25,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201065160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:25,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201065166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:25,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:25,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:25,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:25,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:25,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201065282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201065285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201065285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:25,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:25,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:25,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:25,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:25,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201065501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:25,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201065512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:25,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201065513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:25,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:25,537 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/ee63d87def49436aa3da1e792816c8a8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ee63d87def49436aa3da1e792816c8a8 2024-12-03T04:43:25,546 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/C of 2444c4148e3a1c831bc6f992044b0533 into ee63d87def49436aa3da1e792816c8a8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:25,546 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:25,546 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/C, priority=13, startTime=1733201004844; duration=0sec 2024-12-03T04:43:25,547 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:25,547 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:C 2024-12-03T04:43:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:43:25,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/e48c3fac3d544d0d85823a42655cc1f7 2024-12-03T04:43:25,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/a350682f84e44b91b9444e38560c554c is 50, key is test_row_0/C:col10/1733201003695/Put/seqid=0 2024-12-03T04:43:25,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741896_1072 (size=12301) 2024-12-03T04:43:25,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/a350682f84e44b91b9444e38560c554c 2024-12-03T04:43:25,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/478a0af6c6b94920960d4f0b1b73b8ca as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/478a0af6c6b94920960d4f0b1b73b8ca 2024-12-03T04:43:25,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/478a0af6c6b94920960d4f0b1b73b8ca, entries=200, sequenceid=366, filesize=14.4 K 2024-12-03T04:43:25,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/e48c3fac3d544d0d85823a42655cc1f7 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/e48c3fac3d544d0d85823a42655cc1f7 2024-12-03T04:43:25,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/e48c3fac3d544d0d85823a42655cc1f7, entries=150, sequenceid=366, filesize=12.0 K 2024-12-03T04:43:25,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/a350682f84e44b91b9444e38560c554c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a350682f84e44b91b9444e38560c554c 2024-12-03T04:43:25,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a350682f84e44b91b9444e38560c554c, entries=150, sequenceid=366, filesize=12.0 K 2024-12-03T04:43:25,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=143.13 KB/146560 for 2444c4148e3a1c831bc6f992044b0533 in 815ms, sequenceid=366, compaction requested=false 2024-12-03T04:43:25,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:25,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T04:43:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:25,691 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=143.13 KB heapSize=391.23 KB 2024-12-03T04:43:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/5b6b05f585bd432fad92515bcb5c64ab is 50, key is test_row_0/A:col10/1733201005160/Put/seqid=0 2024-12-03T04:43:25,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741897_1073 (size=12301) 2024-12-03T04:43:25,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:25,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201065879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:25,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201065880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:25,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201065894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201066001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201066001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201066012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,176 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/5b6b05f585bd432fad92515bcb5c64ab 2024-12-03T04:43:26,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/ca9d81f178f64c7596e4c6826007b2dc is 50, key is test_row_0/B:col10/1733201005160/Put/seqid=0 2024-12-03T04:43:26,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201066222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201066222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201066233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741898_1074 (size=12301) 2024-12-03T04:43:26,261 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/ca9d81f178f64c7596e4c6826007b2dc 2024-12-03T04:43:26,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/e5bcd9191ef1476a9bf14af7c6553a73 is 50, key is test_row_0/C:col10/1733201005160/Put/seqid=0 2024-12-03T04:43:26,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741899_1075 (size=12301) 2024-12-03T04:43:26,366 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/e5bcd9191ef1476a9bf14af7c6553a73 2024-12-03T04:43:26,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/5b6b05f585bd432fad92515bcb5c64ab as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/5b6b05f585bd432fad92515bcb5c64ab 2024-12-03T04:43:26,387 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/5b6b05f585bd432fad92515bcb5c64ab, entries=150, sequenceid=403, filesize=12.0 K 2024-12-03T04:43:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/ca9d81f178f64c7596e4c6826007b2dc as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/ca9d81f178f64c7596e4c6826007b2dc 2024-12-03T04:43:26,403 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/ca9d81f178f64c7596e4c6826007b2dc, entries=150, sequenceid=403, filesize=12.0 K 2024-12-03T04:43:26,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/e5bcd9191ef1476a9bf14af7c6553a73 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e5bcd9191ef1476a9bf14af7c6553a73 2024-12-03T04:43:26,420 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e5bcd9191ef1476a9bf14af7c6553a73, entries=150, sequenceid=403, filesize=12.0 K 2024-12-03T04:43:26,423 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~391.19 KB/400576, currentSize=46.96 KB/48090 for 2444c4148e3a1c831bc6f992044b0533 in 732ms, sequenceid=403, compaction requested=true 2024-12-03T04:43:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-03T04:43:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-03T04:43:26,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-03T04:43:26,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0290 sec 2024-12-03T04:43:26,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 3.0370 sec 2024-12-03T04:43:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:26,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T04:43:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:26,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/cd067e49b6fb4f6ead23d19789f85959 is 50, key is test_row_0/A:col10/1733201005889/Put/seqid=0 2024-12-03T04:43:26,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741900_1076 (size=12301) 2024-12-03T04:43:26,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/cd067e49b6fb4f6ead23d19789f85959 2024-12-03T04:43:26,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/4efa219712034ea2af2ac252b7faaf2f is 50, key is test_row_0/B:col10/1733201005889/Put/seqid=0 2024-12-03T04:43:26,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741901_1077 (size=12301) 2024-12-03T04:43:26,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/4efa219712034ea2af2ac252b7faaf2f 2024-12-03T04:43:26,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201066708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/09009144a7794f6e9d1738a0b2097380 is 50, key is test_row_0/C:col10/1733201005889/Put/seqid=0 2024-12-03T04:43:26,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201006665, max timestamp=1733201006715Num uniques 859; 2024-12-03T04:43:26,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:26,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201006665, max timestamp=1733201006715Num uniques 859; 2024-12-03T04:43:26,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:26,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201006665, max timestamp=1733201006715Num uniques 859; 2024-12-03T04:43:26,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:26,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741902_1078 (size=12301) 2024-12-03T04:43:26,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201066829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201066843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201066849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201066952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201066982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201066982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201067163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/09009144a7794f6e9d1738a0b2097380 2024-12-03T04:43:27,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201067198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201067198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/cd067e49b6fb4f6ead23d19789f85959 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/cd067e49b6fb4f6ead23d19789f85959 2024-12-03T04:43:27,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/cd067e49b6fb4f6ead23d19789f85959, entries=150, sequenceid=414, filesize=12.0 K 2024-12-03T04:43:27,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/4efa219712034ea2af2ac252b7faaf2f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/4efa219712034ea2af2ac252b7faaf2f 2024-12-03T04:43:27,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/4efa219712034ea2af2ac252b7faaf2f, entries=150, sequenceid=414, filesize=12.0 K 2024-12-03T04:43:27,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/09009144a7794f6e9d1738a0b2097380 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/09009144a7794f6e9d1738a0b2097380 2024-12-03T04:43:27,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/09009144a7794f6e9d1738a0b2097380, entries=150, sequenceid=414, filesize=12.0 K 2024-12-03T04:43:27,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 2444c4148e3a1c831bc6f992044b0533 in 719ms, sequenceid=414, compaction requested=true 2024-12-03T04:43:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:27,265 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:27,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T04:43:27,265 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:27,267 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:27,267 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/B is initiating minor compaction (all files) 2024-12-03T04:43:27,267 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/B in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:27,267 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/083c0bba7df043f8a2df21cd9ec64279, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/e48c3fac3d544d0d85823a42655cc1f7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/ca9d81f178f64c7596e4c6826007b2dc, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/4efa219712034ea2af2ac252b7faaf2f] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=48.7 K 2024-12-03T04:43:27,268 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:27,268 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/A is initiating minor compaction (all files) 2024-12-03T04:43:27,268 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/A in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:27,268 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/b4cd6d41b46c48d6992ce840c889c176, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/478a0af6c6b94920960d4f0b1b73b8ca, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/5b6b05f585bd432fad92515bcb5c64ab, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/cd067e49b6fb4f6ead23d19789f85959] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=51.0 K 2024-12-03T04:43:27,268 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 083c0bba7df043f8a2df21cd9ec64279, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733201003651 2024-12-03T04:43:27,272 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4cd6d41b46c48d6992ce840c889c176, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733201003651 2024-12-03T04:43:27,273 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e48c3fac3d544d0d85823a42655cc1f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733201003695 2024-12-03T04:43:27,273 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 478a0af6c6b94920960d4f0b1b73b8ca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733201003687 2024-12-03T04:43:27,273 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ca9d81f178f64c7596e4c6826007b2dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733201005122 2024-12-03T04:43:27,273 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b6b05f585bd432fad92515bcb5c64ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733201005122 2024-12-03T04:43:27,274 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 4efa219712034ea2af2ac252b7faaf2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733201005852 2024-12-03T04:43:27,274 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd067e49b6fb4f6ead23d19789f85959, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733201005852 2024-12-03T04:43:27,306 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:27,307 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/2466f27cd944455299b09ee44f927800 is 50, key is test_row_0/A:col10/1733201005889/Put/seqid=0 2024-12-03T04:43:27,316 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:27,317 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/89bb62d4688b4a4fa85a10c0a17be714 is 50, key is test_row_0/B:col10/1733201005889/Put/seqid=0 2024-12-03T04:43:27,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741904_1080 (size=13051) 2024-12-03T04:43:27,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741903_1079 (size=13051) 2024-12-03T04:43:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:27,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T04:43:27,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:27,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:27,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:27,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:27,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:27,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:27,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/7d590cf6aba445289fb1beb37779332b is 50, key is test_row_0/A:col10/1733201007477/Put/seqid=0 2024-12-03T04:43:27,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741905_1081 (size=14741) 2024-12-03T04:43:27,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/7d590cf6aba445289fb1beb37779332b 2024-12-03T04:43:27,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201067525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/408b9b7e37de4b46a3535d880ea126d3 is 50, key is test_row_0/B:col10/1733201007477/Put/seqid=0 2024-12-03T04:43:27,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201067527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201067528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741906_1082 (size=12301) 2024-12-03T04:43:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T04:43:27,562 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:27,563 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-03T04:43:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:43:27,566 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:27,567 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:27,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:27,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201067631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201067652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201067652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:43:27,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:43:27,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:27,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:27,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:27,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:27,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:27,770 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/89bb62d4688b4a4fa85a10c0a17be714 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/89bb62d4688b4a4fa85a10c0a17be714 2024-12-03T04:43:27,799 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/B of 2444c4148e3a1c831bc6f992044b0533 into 89bb62d4688b4a4fa85a10c0a17be714(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:27,799 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:27,799 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/B, priority=12, startTime=1733201007265; duration=0sec 2024-12-03T04:43:27,799 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:27,799 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:B 2024-12-03T04:43:27,800 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:27,801 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/2466f27cd944455299b09ee44f927800 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/2466f27cd944455299b09ee44f927800 2024-12-03T04:43:27,810 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:27,810 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/C is initiating minor compaction (all files) 2024-12-03T04:43:27,810 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/C in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:27,811 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ee63d87def49436aa3da1e792816c8a8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a350682f84e44b91b9444e38560c554c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e5bcd9191ef1476a9bf14af7c6553a73, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/09009144a7794f6e9d1738a0b2097380] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=48.7 K 2024-12-03T04:43:27,812 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ee63d87def49436aa3da1e792816c8a8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733201003651 2024-12-03T04:43:27,812 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a350682f84e44b91b9444e38560c554c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733201003695 2024-12-03T04:43:27,813 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e5bcd9191ef1476a9bf14af7c6553a73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733201005122 2024-12-03T04:43:27,814 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 09009144a7794f6e9d1738a0b2097380, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733201005852 2024-12-03T04:43:27,823 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/A of 2444c4148e3a1c831bc6f992044b0533 into 2466f27cd944455299b09ee44f927800(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:27,823 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:27,823 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/A, priority=12, startTime=1733201007265; duration=0sec 2024-12-03T04:43:27,823 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:27,823 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:A 2024-12-03T04:43:27,834 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#C#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:27,835 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/64b532c76a3d44d5b1a7480b4692ae3c is 50, key is test_row_0/C:col10/1733201005889/Put/seqid=0 2024-12-03T04:43:27,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201067851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741907_1083 (size=13051) 2024-12-03T04:43:27,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201067862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201067862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:27,877 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/64b532c76a3d44d5b1a7480b4692ae3c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/64b532c76a3d44d5b1a7480b4692ae3c 2024-12-03T04:43:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:43:27,885 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/C of 2444c4148e3a1c831bc6f992044b0533 into 64b532c76a3d44d5b1a7480b4692ae3c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:27,885 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:27,885 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/C, priority=12, startTime=1733201007265; duration=0sec 2024-12-03T04:43:27,885 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:27,885 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:C 2024-12-03T04:43:27,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:43:27,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:27,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:27,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:27,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:27,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:27,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/408b9b7e37de4b46a3535d880ea126d3 2024-12-03T04:43:27,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/57931cab71d24620ac01be9f4a3ac69d is 50, key is test_row_0/C:col10/1733201007477/Put/seqid=0 2024-12-03T04:43:28,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741908_1084 (size=12301) 2024-12-03T04:43:28,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/57931cab71d24620ac01be9f4a3ac69d 2024-12-03T04:43:28,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/7d590cf6aba445289fb1beb37779332b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/7d590cf6aba445289fb1beb37779332b 2024-12-03T04:43:28,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/7d590cf6aba445289fb1beb37779332b, entries=200, sequenceid=451, filesize=14.4 K 2024-12-03T04:43:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/408b9b7e37de4b46a3535d880ea126d3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/408b9b7e37de4b46a3535d880ea126d3 2024-12-03T04:43:28,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/408b9b7e37de4b46a3535d880ea126d3, entries=150, sequenceid=451, filesize=12.0 K 2024-12-03T04:43:28,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/57931cab71d24620ac01be9f4a3ac69d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/57931cab71d24620ac01be9f4a3ac69d 2024-12-03T04:43:28,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:43:28,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:28,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/57931cab71d24620ac01be9f4a3ac69d, entries=150, sequenceid=451, filesize=12.0 K 2024-12-03T04:43:28,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 2444c4148e3a1c831bc6f992044b0533 in 577ms, sequenceid=451, compaction requested=false 2024-12-03T04:43:28,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:28,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:43:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:28,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:28,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:28,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:28,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/254c4dea9ddc462c93ff055b5a2cd263 is 50, key is test_row_0/A:col10/1733201007525/Put/seqid=0 2024-12-03T04:43:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:43:28,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:43:28,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741909_1085 (size=14741) 2024-12-03T04:43:28,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/254c4dea9ddc462c93ff055b5a2cd263 2024-12-03T04:43:28,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/d09f82e2dd734cea9bb13850253b4ceb is 50, key is test_row_0/B:col10/1733201007525/Put/seqid=0 2024-12-03T04:43:28,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201008203, max timestamp=1733201008256Num uniques 859; 2024-12-03T04:43:28,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:28,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201008203, max timestamp=1733201008256Num uniques 859; 2024-12-03T04:43:28,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:28,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741910_1086 (size=12301) 2024-12-03T04:43:28,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733201008203, max timestamp=1733201008256Num uniques 859; 2024-12-03T04:43:28,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T04:43:28,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/d09f82e2dd734cea9bb13850253b4ceb 2024-12-03T04:43:28,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/c7d60cc816fc44ce908624875c243e51 is 50, key is test_row_0/C:col10/1733201007525/Put/seqid=0 2024-12-03T04:43:28,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:43:28,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:28,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741911_1087 (size=12301) 2024-12-03T04:43:28,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201068375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:28,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201068383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:28,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201068386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:28,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201068492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:28,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201068492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:28,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201068501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:28,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:43:28,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:28,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:43:28,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:28,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:43:28,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201068702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:28,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201068702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:28,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201068725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:28,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/c7d60cc816fc44ce908624875c243e51 2024-12-03T04:43:28,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/254c4dea9ddc462c93ff055b5a2cd263 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/254c4dea9ddc462c93ff055b5a2cd263 2024-12-03T04:43:28,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/254c4dea9ddc462c93ff055b5a2cd263, entries=200, sequenceid=464, filesize=14.4 K 2024-12-03T04:43:28,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/d09f82e2dd734cea9bb13850253b4ceb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/d09f82e2dd734cea9bb13850253b4ceb 2024-12-03T04:43:28,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/d09f82e2dd734cea9bb13850253b4ceb, entries=150, sequenceid=464, filesize=12.0 K 2024-12-03T04:43:28,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/c7d60cc816fc44ce908624875c243e51 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c7d60cc816fc44ce908624875c243e51 2024-12-03T04:43:28,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:43:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c7d60cc816fc44ce908624875c243e51, entries=150, sequenceid=464, filesize=12.0 K 2024-12-03T04:43:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:28,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for 2444c4148e3a1c831bc6f992044b0533 in 673ms, sequenceid=464, compaction requested=true 2024-12-03T04:43:28,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:28,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:28,843 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:28,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:28,843 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:28,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:28,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:28,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:28,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:28,845 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:28,846 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/A is initiating minor compaction (all files) 2024-12-03T04:43:28,846 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/A in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,846 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/2466f27cd944455299b09ee44f927800, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/7d590cf6aba445289fb1beb37779332b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/254c4dea9ddc462c93ff055b5a2cd263] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=41.5 K 2024-12-03T04:43:28,846 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:28,846 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/B is initiating minor compaction (all files) 2024-12-03T04:43:28,846 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2466f27cd944455299b09ee44f927800, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733201005852 2024-12-03T04:43:28,846 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/B in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,846 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/89bb62d4688b4a4fa85a10c0a17be714, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/408b9b7e37de4b46a3535d880ea126d3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/d09f82e2dd734cea9bb13850253b4ceb] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=36.8 K 2024-12-03T04:43:28,847 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 89bb62d4688b4a4fa85a10c0a17be714, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733201005852 2024-12-03T04:43:28,847 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d590cf6aba445289fb1beb37779332b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733201006800 2024-12-03T04:43:28,848 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 254c4dea9ddc462c93ff055b5a2cd263, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733201007502 2024-12-03T04:43:28,848 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 408b9b7e37de4b46a3535d880ea126d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733201006800 2024-12-03T04:43:28,849 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting d09f82e2dd734cea9bb13850253b4ceb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733201007502 2024-12-03T04:43:28,868 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:28,869 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:28,870 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/f6d9e9cb68534b11b4a8e6340e2e9896 is 50, key is test_row_0/A:col10/1733201007525/Put/seqid=0 2024-12-03T04:43:28,871 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/90bfb863394348c8b8f6dfbd0e8ff552 is 50, key is test_row_0/B:col10/1733201007525/Put/seqid=0 2024-12-03T04:43:28,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741913_1089 (size=13153) 2024-12-03T04:43:28,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741912_1088 (size=13153) 2024-12-03T04:43:28,892 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/90bfb863394348c8b8f6dfbd0e8ff552 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/90bfb863394348c8b8f6dfbd0e8ff552 2024-12-03T04:43:28,902 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/B of 2444c4148e3a1c831bc6f992044b0533 into 90bfb863394348c8b8f6dfbd0e8ff552(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:28,902 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:28,902 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/B, priority=13, startTime=1733201008843; duration=0sec 2024-12-03T04:43:28,903 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:28,903 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:B 2024-12-03T04:43:28,903 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:28,904 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:28,904 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/C is initiating minor compaction (all files) 2024-12-03T04:43:28,904 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/C in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,904 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/64b532c76a3d44d5b1a7480b4692ae3c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/57931cab71d24620ac01be9f4a3ac69d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c7d60cc816fc44ce908624875c243e51] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=36.8 K 2024-12-03T04:43:28,904 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 64b532c76a3d44d5b1a7480b4692ae3c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733201005852 2024-12-03T04:43:28,905 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 57931cab71d24620ac01be9f4a3ac69d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733201006800 2024-12-03T04:43:28,905 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c7d60cc816fc44ce908624875c243e51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733201007502 2024-12-03T04:43:28,925 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#C#compaction#77 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:28,926 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/9252969e8511400f8242694a6c76e484 is 50, key is test_row_0/C:col10/1733201007525/Put/seqid=0 2024-12-03T04:43:28,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741914_1090 (size=13153) 2024-12-03T04:43:28,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T04:43:28,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:28,998 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-12-03T04:43:28,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:28,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:28,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:28,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:28,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:28,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:29,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:29,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/9b5ac3e85694417aabd373b6c8740f0f is 50, key is test_row_0/A:col10/1733201008356/Put/seqid=0 2024-12-03T04:43:29,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201069043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201069044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201069051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741915_1091 (size=12301) 2024-12-03T04:43:29,075 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/9b5ac3e85694417aabd373b6c8740f0f 2024-12-03T04:43:29,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/2939e9603a934c55b674f4065ac25f80 is 50, key is test_row_0/B:col10/1733201008356/Put/seqid=0 2024-12-03T04:43:29,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741916_1092 (size=12301) 2024-12-03T04:43:29,148 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/2939e9603a934c55b674f4065ac25f80 2024-12-03T04:43:29,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201069162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201069162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/0ce55feb876249598a6239b1b8cb76bb is 50, key is test_row_0/C:col10/1733201008356/Put/seqid=0 2024-12-03T04:43:29,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201069171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741917_1093 (size=12301) 2024-12-03T04:43:29,299 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/f6d9e9cb68534b11b4a8e6340e2e9896 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/f6d9e9cb68534b11b4a8e6340e2e9896 2024-12-03T04:43:29,312 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/A of 2444c4148e3a1c831bc6f992044b0533 into f6d9e9cb68534b11b4a8e6340e2e9896(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:29,312 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:29,312 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/A, priority=13, startTime=1733201008843; duration=0sec 2024-12-03T04:43:29,312 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:29,312 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:A 2024-12-03T04:43:29,349 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/9252969e8511400f8242694a6c76e484 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/9252969e8511400f8242694a6c76e484 2024-12-03T04:43:29,361 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/C of 2444c4148e3a1c831bc6f992044b0533 into 9252969e8511400f8242694a6c76e484(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:29,361 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:29,361 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/C, priority=13, startTime=1733201008844; duration=0sec 2024-12-03T04:43:29,361 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:29,361 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:C 2024-12-03T04:43:29,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201069372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201069374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201069401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,604 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/0ce55feb876249598a6239b1b8cb76bb 2024-12-03T04:43:29,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/9b5ac3e85694417aabd373b6c8740f0f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9b5ac3e85694417aabd373b6c8740f0f 2024-12-03T04:43:29,647 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9b5ac3e85694417aabd373b6c8740f0f, entries=150, sequenceid=502, filesize=12.0 K 2024-12-03T04:43:29,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/2939e9603a934c55b674f4065ac25f80 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/2939e9603a934c55b674f4065ac25f80 2024-12-03T04:43:29,656 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/2939e9603a934c55b674f4065ac25f80, entries=150, sequenceid=502, filesize=12.0 K 2024-12-03T04:43:29,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/0ce55feb876249598a6239b1b8cb76bb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/0ce55feb876249598a6239b1b8cb76bb 2024-12-03T04:43:29,672 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/0ce55feb876249598a6239b1b8cb76bb, entries=150, sequenceid=502, filesize=12.0 K 2024-12-03T04:43:29,680 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=46.96 KB/48090 for 2444c4148e3a1c831bc6f992044b0533 in 682ms, sequenceid=502, compaction requested=false 2024-12-03T04:43:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-03T04:43:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-03T04:43:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:43:29,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T04:43:29,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T04:43:29,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1460 sec 2024-12-03T04:43:29,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:29,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:29,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:29,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:29,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:29,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:29,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.1770 sec 2024-12-03T04:43:29,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/b759460e60074868bd3765bdb09a55ea is 50, key is test_row_0/A:col10/1733201009042/Put/seqid=0 2024-12-03T04:43:29,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741918_1094 (size=12301) 2024-12-03T04:43:29,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201069862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201069862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201069863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201069972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201069983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:29,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201069983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201070182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201070191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201070202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/b759460e60074868bd3765bdb09a55ea 2024-12-03T04:43:30,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/124f827cb3864155adb85dd553786463 is 50, key is test_row_0/B:col10/1733201009042/Put/seqid=0 2024-12-03T04:43:30,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741919_1095 (size=12301) 2024-12-03T04:43:30,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/124f827cb3864155adb85dd553786463 2024-12-03T04:43:30,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/f49897fcbefd4445bc57bf37c96ae4fb is 50, key is test_row_0/C:col10/1733201009042/Put/seqid=0 2024-12-03T04:43:30,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741920_1096 (size=12301) 2024-12-03T04:43:30,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/f49897fcbefd4445bc57bf37c96ae4fb 2024-12-03T04:43:30,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/b759460e60074868bd3765bdb09a55ea as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/b759460e60074868bd3765bdb09a55ea 2024-12-03T04:43:30,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/b759460e60074868bd3765bdb09a55ea, entries=150, sequenceid=519, filesize=12.0 K 2024-12-03T04:43:30,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/124f827cb3864155adb85dd553786463 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/124f827cb3864155adb85dd553786463 2024-12-03T04:43:30,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/124f827cb3864155adb85dd553786463, entries=150, sequenceid=519, filesize=12.0 K 2024-12-03T04:43:30,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/f49897fcbefd4445bc57bf37c96ae4fb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f49897fcbefd4445bc57bf37c96ae4fb 2024-12-03T04:43:30,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f49897fcbefd4445bc57bf37c96ae4fb, entries=150, sequenceid=519, filesize=12.0 K 2024-12-03T04:43:30,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2444c4148e3a1c831bc6f992044b0533 in 703ms, sequenceid=519, compaction requested=true 2024-12-03T04:43:30,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:30,440 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:30,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:30,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:30,441 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:30,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:30,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:30,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:30,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:30,442 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:30,443 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/A is initiating minor compaction (all files) 2024-12-03T04:43:30,443 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/A in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:30,443 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/f6d9e9cb68534b11b4a8e6340e2e9896, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9b5ac3e85694417aabd373b6c8740f0f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/b759460e60074868bd3765bdb09a55ea] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=36.9 K 2024-12-03T04:43:30,443 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:30,443 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/B is initiating minor compaction (all files) 2024-12-03T04:43:30,443 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/B in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:30,443 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/90bfb863394348c8b8f6dfbd0e8ff552, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/2939e9603a934c55b674f4065ac25f80, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/124f827cb3864155adb85dd553786463] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=36.9 K 2024-12-03T04:43:30,445 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6d9e9cb68534b11b4a8e6340e2e9896, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733201007502 2024-12-03T04:43:30,445 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 90bfb863394348c8b8f6dfbd0e8ff552, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733201007502 2024-12-03T04:43:30,446 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b5ac3e85694417aabd373b6c8740f0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733201008356 2024-12-03T04:43:30,448 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting b759460e60074868bd3765bdb09a55ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733201009042 2024-12-03T04:43:30,448 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 2939e9603a934c55b674f4065ac25f80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733201008356 2024-12-03T04:43:30,449 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 124f827cb3864155adb85dd553786463, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733201009042 2024-12-03T04:43:30,470 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:30,470 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/86c649189d77447e80f6903de27f9946 is 50, key is test_row_0/A:col10/1733201009042/Put/seqid=0 2024-12-03T04:43:30,475 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#B#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:30,475 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/0b47ef060b8347928d33f68139378647 is 50, key is test_row_0/B:col10/1733201009042/Put/seqid=0 2024-12-03T04:43:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:30,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T04:43:30,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:30,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:30,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:30,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:30,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:30,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:30,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741921_1097 (size=13255) 2024-12-03T04:43:30,544 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/86c649189d77447e80f6903de27f9946 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/86c649189d77447e80f6903de27f9946 2024-12-03T04:43:30,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741922_1098 (size=13255) 2024-12-03T04:43:30,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/178f327cb875477fb6fdd283eff8b614 is 50, key is test_row_0/A:col10/1733201010497/Put/seqid=0 2024-12-03T04:43:30,559 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/A of 2444c4148e3a1c831bc6f992044b0533 into 86c649189d77447e80f6903de27f9946(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:30,560 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:30,560 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/A, priority=13, startTime=1733201010440; duration=0sec 2024-12-03T04:43:30,560 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:30,560 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:A 2024-12-03T04:43:30,560 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:30,563 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:30,563 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/C is initiating minor compaction (all files) 2024-12-03T04:43:30,563 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/C in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:30,564 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/9252969e8511400f8242694a6c76e484, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/0ce55feb876249598a6239b1b8cb76bb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f49897fcbefd4445bc57bf37c96ae4fb] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=36.9 K 2024-12-03T04:43:30,570 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9252969e8511400f8242694a6c76e484, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733201007502 2024-12-03T04:43:30,571 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ce55feb876249598a6239b1b8cb76bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733201008356 2024-12-03T04:43:30,571 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f49897fcbefd4445bc57bf37c96ae4fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733201009042 2024-12-03T04:43:30,574 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/0b47ef060b8347928d33f68139378647 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/0b47ef060b8347928d33f68139378647 2024-12-03T04:43:30,587 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/B of 2444c4148e3a1c831bc6f992044b0533 into 0b47ef060b8347928d33f68139378647(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:30,587 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:30,587 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/B, priority=13, startTime=1733201010441; duration=0sec 2024-12-03T04:43:30,587 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:30,587 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:B 2024-12-03T04:43:30,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741923_1099 (size=14741) 2024-12-03T04:43:30,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/178f327cb875477fb6fdd283eff8b614 2024-12-03T04:43:30,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201070579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201070583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201070597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,623 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#C#compaction#87 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:30,623 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/40ea5c7d461a4634854f152bf838da77 is 50, key is test_row_0/C:col10/1733201009042/Put/seqid=0 2024-12-03T04:43:30,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/8ced8e24743a4044b0bd63868fe3f12c is 50, key is test_row_0/B:col10/1733201010497/Put/seqid=0 2024-12-03T04:43:30,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741924_1100 (size=13255) 2024-12-03T04:43:30,700 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/40ea5c7d461a4634854f152bf838da77 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/40ea5c7d461a4634854f152bf838da77 2024-12-03T04:43:30,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741925_1101 (size=12301) 2024-12-03T04:43:30,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201070702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,713 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/C of 2444c4148e3a1c831bc6f992044b0533 into 40ea5c7d461a4634854f152bf838da77(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:30,713 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:30,713 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/C, priority=13, startTime=1733201010442; duration=0sec 2024-12-03T04:43:30,714 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:30,714 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:C 2024-12-03T04:43:30,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201070702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/8ced8e24743a4044b0bd63868fe3f12c 2024-12-03T04:43:30,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201070719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/e9aca328651649cb8943d240925e2624 is 50, key is test_row_0/C:col10/1733201010497/Put/seqid=0 2024-12-03T04:43:30,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741926_1102 (size=12301) 2024-12-03T04:43:30,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/e9aca328651649cb8943d240925e2624 2024-12-03T04:43:30,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/178f327cb875477fb6fdd283eff8b614 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/178f327cb875477fb6fdd283eff8b614 2024-12-03T04:43:30,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/178f327cb875477fb6fdd283eff8b614, entries=200, sequenceid=537, filesize=14.4 K 2024-12-03T04:43:30,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/8ced8e24743a4044b0bd63868fe3f12c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/8ced8e24743a4044b0bd63868fe3f12c 2024-12-03T04:43:30,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/8ced8e24743a4044b0bd63868fe3f12c, entries=150, sequenceid=537, filesize=12.0 K 2024-12-03T04:43:30,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/e9aca328651649cb8943d240925e2624 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e9aca328651649cb8943d240925e2624 2024-12-03T04:43:30,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e9aca328651649cb8943d240925e2624, entries=150, sequenceid=537, filesize=12.0 K 2024-12-03T04:43:30,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2444c4148e3a1c831bc6f992044b0533 in 302ms, sequenceid=537, compaction requested=false 2024-12-03T04:43:30,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:30,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T04:43:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:30,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:30,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:30,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/561bfec4f7c24e888d95dc13d5091650 is 50, key is test_row_0/A:col10/1733201010572/Put/seqid=0 2024-12-03T04:43:30,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741927_1103 (size=12301) 2024-12-03T04:43:30,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201070972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201070978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:30,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201070980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:31,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201071094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:31,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201071094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:31,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201071103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:31,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:31,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:31,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201071312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201071312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:31,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201071322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:31,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/561bfec4f7c24e888d95dc13d5091650 2024-12-03T04:43:31,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/224d40642828493a8c8d33db06b7c81d is 50, key is test_row_0/B:col10/1733201010572/Put/seqid=0 2024-12-03T04:43:31,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741928_1104 (size=12301) 2024-12-03T04:43:31,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201071622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201071622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:31,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201071632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T04:43:31,742 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:31,743 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-03T04:43:31,750 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:43:31,753 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:31,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:31,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/224d40642828493a8c8d33db06b7c81d 2024-12-03T04:43:31,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/5947c08c054449b6a98c2c2f109ad298 is 50, key is test_row_0/C:col10/1733201010572/Put/seqid=0 2024-12-03T04:43:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:43:31,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741929_1105 (size=12301) 2024-12-03T04:43:31,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:43:31,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:31,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:32,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:43:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:32,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:32,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:43:32,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46502 deadline: 1733201072144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:32,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:32,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46514 deadline: 1733201072144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46518 deadline: 1733201072144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:43:32,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:43:32,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:32,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:32,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:32,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:32,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/5947c08c054449b6a98c2c2f109ad298 2024-12-03T04:43:32,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/561bfec4f7c24e888d95dc13d5091650 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/561bfec4f7c24e888d95dc13d5091650 2024-12-03T04:43:32,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/561bfec4f7c24e888d95dc13d5091650, entries=150, sequenceid=556, filesize=12.0 K 2024-12-03T04:43:32,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/224d40642828493a8c8d33db06b7c81d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/224d40642828493a8c8d33db06b7c81d 2024-12-03T04:43:32,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/224d40642828493a8c8d33db06b7c81d, entries=150, sequenceid=556, filesize=12.0 K 2024-12-03T04:43:32,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/5947c08c054449b6a98c2c2f109ad298 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/5947c08c054449b6a98c2c2f109ad298 2024-12-03T04:43:32,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/5947c08c054449b6a98c2c2f109ad298, entries=150, sequenceid=556, filesize=12.0 K 2024-12-03T04:43:32,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2444c4148e3a1c831bc6f992044b0533 in 1429ms, sequenceid=556, compaction requested=true 2024-12-03T04:43:32,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:32,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:32,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:32,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:32,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T04:43:32,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2444c4148e3a1c831bc6f992044b0533:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:32,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T04:43:32,355 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:32,357 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:32,357 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/A is initiating minor compaction (all files) 2024-12-03T04:43:32,357 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/A in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:32,357 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:32,357 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/86c649189d77447e80f6903de27f9946, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/178f327cb875477fb6fdd283eff8b614, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/561bfec4f7c24e888d95dc13d5091650] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=39.4 K 2024-12-03T04:43:32,358 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c649189d77447e80f6903de27f9946, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733201009042 2024-12-03T04:43:32,359 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 178f327cb875477fb6fdd283eff8b614, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1733201009795 2024-12-03T04:43:32,359 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:32,359 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/C is initiating minor compaction (all files) 2024-12-03T04:43:32,359 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/C in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:32,360 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/40ea5c7d461a4634854f152bf838da77, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e9aca328651649cb8943d240925e2624, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/5947c08c054449b6a98c2c2f109ad298] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=37.0 K 2024-12-03T04:43:32,360 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 561bfec4f7c24e888d95dc13d5091650, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1733201010572 2024-12-03T04:43:32,361 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40ea5c7d461a4634854f152bf838da77, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733201009042 2024-12-03T04:43:32,362 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9aca328651649cb8943d240925e2624, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1733201009795 2024-12-03T04:43:32,363 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5947c08c054449b6a98c2c2f109ad298, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1733201010572 2024-12-03T04:43:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:43:32,383 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#A#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:32,384 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/d3bea4d15a8a48e4b8d8690896be640b is 50, key is test_row_0/A:col10/1733201010572/Put/seqid=0 2024-12-03T04:43:32,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T04:43:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:32,386 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T04:43:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:32,395 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#C#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:32,396 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/204fe4e247834bcfa67bbd5b7912d9b8 is 50, key is test_row_0/C:col10/1733201010572/Put/seqid=0 2024-12-03T04:43:32,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/9d0b5f58111f48a794793c59be1ed171 is 50, key is test_row_0/A:col10/1733201010967/Put/seqid=0 2024-12-03T04:43:32,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741930_1106 (size=13357) 2024-12-03T04:43:32,472 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/d3bea4d15a8a48e4b8d8690896be640b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/d3bea4d15a8a48e4b8d8690896be640b 2024-12-03T04:43:32,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741931_1107 (size=13357) 2024-12-03T04:43:32,495 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/A of 2444c4148e3a1c831bc6f992044b0533 into d3bea4d15a8a48e4b8d8690896be640b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:32,495 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:32,495 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/A, priority=13, startTime=1733201012354; duration=0sec 2024-12-03T04:43:32,495 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:32,496 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:A 2024-12-03T04:43:32,496 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:32,498 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:32,498 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 2444c4148e3a1c831bc6f992044b0533/B is initiating minor compaction (all files) 2024-12-03T04:43:32,499 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2444c4148e3a1c831bc6f992044b0533/B in TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:32,499 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/0b47ef060b8347928d33f68139378647, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/8ced8e24743a4044b0bd63868fe3f12c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/224d40642828493a8c8d33db06b7c81d] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp, totalSize=37.0 K 2024-12-03T04:43:32,499 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b47ef060b8347928d33f68139378647, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733201009042 2024-12-03T04:43:32,500 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ced8e24743a4044b0bd63868fe3f12c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1733201009795 2024-12-03T04:43:32,500 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 224d40642828493a8c8d33db06b7c81d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1733201010572 2024-12-03T04:43:32,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741932_1108 (size=12301) 2024-12-03T04:43:32,516 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2444c4148e3a1c831bc6f992044b0533#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:32,516 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/9d0b5f58111f48a794793c59be1ed171 2024-12-03T04:43:32,516 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/0f2c2161a42b409ab17faa61f891e842 is 50, key is test_row_0/B:col10/1733201010572/Put/seqid=0 2024-12-03T04:43:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/6d7cf9e758054be0ae3500fbf3b4b2e0 is 50, key is test_row_0/B:col10/1733201010967/Put/seqid=0 2024-12-03T04:43:32,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741933_1109 (size=13357) 2024-12-03T04:43:32,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741934_1110 (size=12301) 2024-12-03T04:43:32,621 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-03T04:43:32,621 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:32,626 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-03T04:43:32,626 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:32,628 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:32,628 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-03T04:43:32,628 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:32,628 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:32,629 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:32,629 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:32,629 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:32,629 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:32,629 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:32,630 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:32,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:32,638 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-03T04:43:32,638 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:32,638 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:32,638 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:32,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:43:32,893 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/204fe4e247834bcfa67bbd5b7912d9b8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/204fe4e247834bcfa67bbd5b7912d9b8 2024-12-03T04:43:32,899 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/C of 2444c4148e3a1c831bc6f992044b0533 into 204fe4e247834bcfa67bbd5b7912d9b8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:32,899 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:32,899 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/C, priority=13, startTime=1733201012354; duration=0sec 2024-12-03T04:43:32,899 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:32,899 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:C 2024-12-03T04:43:32,982 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/0f2c2161a42b409ab17faa61f891e842 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/0f2c2161a42b409ab17faa61f891e842 2024-12-03T04:43:32,989 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2444c4148e3a1c831bc6f992044b0533/B of 2444c4148e3a1c831bc6f992044b0533 into 0f2c2161a42b409ab17faa61f891e842(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:32,990 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:32,990 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., storeName=2444c4148e3a1c831bc6f992044b0533/B, priority=13, startTime=1733201012354; duration=0sec 2024-12-03T04:43:32,990 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:32,990 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2444c4148e3a1c831bc6f992044b0533:B 2024-12-03T04:43:33,012 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/6d7cf9e758054be0ae3500fbf3b4b2e0 2024-12-03T04:43:33,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/7137e538d6eb4e3cb2a0fde2163fce5e is 50, key is test_row_0/C:col10/1733201010967/Put/seqid=0 2024-12-03T04:43:33,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741935_1111 (size=12301) 2024-12-03T04:43:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:33,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. as already flushing 2024-12-03T04:43:33,165 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-03T04:43:33,165 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-03T04:43:33,165 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:33,165 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:33,165 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:33,165 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:33,165 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:33,165 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:33,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:33,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:33,173 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-03T04:43:33,173 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:33,173 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:33,173 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:33,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:33,436 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/7137e538d6eb4e3cb2a0fde2163fce5e 2024-12-03T04:43:33,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/9d0b5f58111f48a794793c59be1ed171 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9d0b5f58111f48a794793c59be1ed171 2024-12-03T04:43:33,449 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9d0b5f58111f48a794793c59be1ed171, entries=150, sequenceid=572, filesize=12.0 K 2024-12-03T04:43:33,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/6d7cf9e758054be0ae3500fbf3b4b2e0 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/6d7cf9e758054be0ae3500fbf3b4b2e0 2024-12-03T04:43:33,460 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/6d7cf9e758054be0ae3500fbf3b4b2e0, entries=150, sequenceid=572, filesize=12.0 K 2024-12-03T04:43:33,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/7137e538d6eb4e3cb2a0fde2163fce5e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/7137e538d6eb4e3cb2a0fde2163fce5e 2024-12-03T04:43:33,466 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/7137e538d6eb4e3cb2a0fde2163fce5e, entries=150, sequenceid=572, filesize=12.0 K 2024-12-03T04:43:33,468 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=20.13 KB/20610 for 2444c4148e3a1c831bc6f992044b0533 in 1081ms, sequenceid=572, compaction requested=false 2024-12-03T04:43:33,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 2444c4148e3a1c831bc6f992044b0533: 2024-12-03T04:43:33,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:33,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-03T04:43:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-03T04:43:33,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T04:43:33,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7170 sec 2024-12-03T04:43:33,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.7300 sec 2024-12-03T04:43:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T04:43:33,904 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:34,649 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-03T04:43:34,649 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:34,649 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:34,649 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:34,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:34,679 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-03T04:43:34,679 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:34,679 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:34,679 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:34,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:34,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 133 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 150 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1444 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1392 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 715 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2144 rows 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 708 2024-12-03T04:43:34,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2124 rows 2024-12-03T04:43:34,680 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T04:43:34,680 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T04:43:34,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:34,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:34,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:34,686 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T04:43:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T04:43:34,700 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201014699"}]},"ts":"1733201014699"} 2024-12-03T04:43:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T04:43:34,714 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T04:43:34,714 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T04:43:34,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T04:43:34,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2444c4148e3a1c831bc6f992044b0533, UNASSIGN}] 2024-12-03T04:43:34,725 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2444c4148e3a1c831bc6f992044b0533, UNASSIGN 2024-12-03T04:43:34,726 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=2444c4148e3a1c831bc6f992044b0533, regionState=CLOSING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:43:34,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2444c4148e3a1c831bc6f992044b0533, UNASSIGN because future has completed 2024-12-03T04:43:34,737 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T04:43:34,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:43:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T04:43:34,898 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:34,898 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T04:43:34,899 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 2444c4148e3a1c831bc6f992044b0533, disabling compactions & flushes 2024-12-03T04:43:34,899 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:34,899 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:34,899 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. after waiting 0 ms 2024-12-03T04:43:34,899 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:34,899 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 2444c4148e3a1c831bc6f992044b0533 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T04:43:34,899 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=A 2024-12-03T04:43:34,900 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:34,900 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=B 2024-12-03T04:43:34,900 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:34,900 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2444c4148e3a1c831bc6f992044b0533, store=C 2024-12-03T04:43:34,900 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:34,916 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/3a95a4a16b5042d6a04346446b74f14c is 50, key is test_row_1/A:col10/1733201013171/Put/seqid=0 2024-12-03T04:43:34,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741936_1112 (size=9857) 2024-12-03T04:43:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T04:43:35,163 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T04:43:35,323 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=583 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/3a95a4a16b5042d6a04346446b74f14c 2024-12-03T04:43:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T04:43:35,343 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/644562c452ef4d63b0287b58485b1531 is 50, key is test_row_1/B:col10/1733201013171/Put/seqid=0 2024-12-03T04:43:35,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741937_1113 (size=9857) 2024-12-03T04:43:35,366 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=583 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/644562c452ef4d63b0287b58485b1531 2024-12-03T04:43:35,379 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/8d7451ee4bbc471395f495d092c499f1 is 50, key is test_row_1/C:col10/1733201013171/Put/seqid=0 2024-12-03T04:43:35,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741938_1114 (size=9857) 2024-12-03T04:43:35,387 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=583 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/8d7451ee4bbc471395f495d092c499f1 2024-12-03T04:43:35,395 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/A/3a95a4a16b5042d6a04346446b74f14c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/3a95a4a16b5042d6a04346446b74f14c 2024-12-03T04:43:35,401 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/3a95a4a16b5042d6a04346446b74f14c, entries=100, sequenceid=583, filesize=9.6 K 2024-12-03T04:43:35,402 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/B/644562c452ef4d63b0287b58485b1531 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/644562c452ef4d63b0287b58485b1531 2024-12-03T04:43:35,409 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/644562c452ef4d63b0287b58485b1531, entries=100, sequenceid=583, filesize=9.6 K 2024-12-03T04:43:35,410 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/.tmp/C/8d7451ee4bbc471395f495d092c499f1 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/8d7451ee4bbc471395f495d092c499f1 2024-12-03T04:43:35,416 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/8d7451ee4bbc471395f495d092c499f1, entries=100, sequenceid=583, filesize=9.6 K 2024-12-03T04:43:35,418 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2444c4148e3a1c831bc6f992044b0533 in 518ms, sequenceid=583, compaction requested=true 2024-12-03T04:43:35,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/e828cac9104947cfad9853660faee996, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a786a70b41a24a90b6c903e3ae30b67c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6bb91d2038da4f55832e964d463d743d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/c4045edf0d4e40cfa88f6c0c79a1092b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/d0c2819fd75844ab95d49a67ff1b7deb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/56ca4a5b6a034b89bc59f48180be9304, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/8e66a1748448474ab325a808ad4e70af, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6d74149b1813457e934d808d517c3ca8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/bfadbfb15a1d4a649b7a1df4119ac7eb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/dae87fc79c54453b8fa1c8275af765cd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/fdc9526507ed410f984b5919aad7a287, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a1ee1c0c97b64c52b1ea498d84d1f6df, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9fbe1425bf75498fb3b398f92d32c80e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/ab3e0e1b8abd46de8ecc01852d0af92c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/30b4975d2a4c433b84b0d9600edf2383, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/21b85f1ff2954daaafb553b5fcf23055, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/8a8be42760de42b1856d7f1254d4d719, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/2f267023e32b4d0b95da9c4653750f93, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/b4cd6d41b46c48d6992ce840c889c176, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/478a0af6c6b94920960d4f0b1b73b8ca, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/5b6b05f585bd432fad92515bcb5c64ab, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/2466f27cd944455299b09ee44f927800, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/cd067e49b6fb4f6ead23d19789f85959, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/7d590cf6aba445289fb1beb37779332b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/254c4dea9ddc462c93ff055b5a2cd263, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/f6d9e9cb68534b11b4a8e6340e2e9896, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9b5ac3e85694417aabd373b6c8740f0f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/86c649189d77447e80f6903de27f9946, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/b759460e60074868bd3765bdb09a55ea, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/178f327cb875477fb6fdd283eff8b614, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/561bfec4f7c24e888d95dc13d5091650] to archive 2024-12-03T04:43:35,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:43:35,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/e828cac9104947cfad9853660faee996 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/e828cac9104947cfad9853660faee996 2024-12-03T04:43:35,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a786a70b41a24a90b6c903e3ae30b67c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a786a70b41a24a90b6c903e3ae30b67c 2024-12-03T04:43:35,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6bb91d2038da4f55832e964d463d743d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6bb91d2038da4f55832e964d463d743d 2024-12-03T04:43:35,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/c4045edf0d4e40cfa88f6c0c79a1092b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/c4045edf0d4e40cfa88f6c0c79a1092b 2024-12-03T04:43:35,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/d0c2819fd75844ab95d49a67ff1b7deb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/d0c2819fd75844ab95d49a67ff1b7deb 2024-12-03T04:43:35,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/56ca4a5b6a034b89bc59f48180be9304 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/56ca4a5b6a034b89bc59f48180be9304 2024-12-03T04:43:35,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/8e66a1748448474ab325a808ad4e70af to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/8e66a1748448474ab325a808ad4e70af 2024-12-03T04:43:35,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6d74149b1813457e934d808d517c3ca8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/6d74149b1813457e934d808d517c3ca8 2024-12-03T04:43:35,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/bfadbfb15a1d4a649b7a1df4119ac7eb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/bfadbfb15a1d4a649b7a1df4119ac7eb 2024-12-03T04:43:35,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/dae87fc79c54453b8fa1c8275af765cd to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/dae87fc79c54453b8fa1c8275af765cd 2024-12-03T04:43:35,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/fdc9526507ed410f984b5919aad7a287 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/fdc9526507ed410f984b5919aad7a287 2024-12-03T04:43:35,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a1ee1c0c97b64c52b1ea498d84d1f6df to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/a1ee1c0c97b64c52b1ea498d84d1f6df 2024-12-03T04:43:35,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9fbe1425bf75498fb3b398f92d32c80e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9fbe1425bf75498fb3b398f92d32c80e 2024-12-03T04:43:35,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/ab3e0e1b8abd46de8ecc01852d0af92c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/ab3e0e1b8abd46de8ecc01852d0af92c 2024-12-03T04:43:35,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/30b4975d2a4c433b84b0d9600edf2383 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/30b4975d2a4c433b84b0d9600edf2383 2024-12-03T04:43:35,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/21b85f1ff2954daaafb553b5fcf23055 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/21b85f1ff2954daaafb553b5fcf23055 2024-12-03T04:43:35,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/8a8be42760de42b1856d7f1254d4d719 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/8a8be42760de42b1856d7f1254d4d719 2024-12-03T04:43:35,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/2f267023e32b4d0b95da9c4653750f93 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/2f267023e32b4d0b95da9c4653750f93 2024-12-03T04:43:35,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/b4cd6d41b46c48d6992ce840c889c176 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/b4cd6d41b46c48d6992ce840c889c176 2024-12-03T04:43:35,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/478a0af6c6b94920960d4f0b1b73b8ca to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/478a0af6c6b94920960d4f0b1b73b8ca 2024-12-03T04:43:35,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/5b6b05f585bd432fad92515bcb5c64ab to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/5b6b05f585bd432fad92515bcb5c64ab 2024-12-03T04:43:35,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/2466f27cd944455299b09ee44f927800 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/2466f27cd944455299b09ee44f927800 2024-12-03T04:43:35,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/cd067e49b6fb4f6ead23d19789f85959 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/cd067e49b6fb4f6ead23d19789f85959 2024-12-03T04:43:35,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/7d590cf6aba445289fb1beb37779332b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/7d590cf6aba445289fb1beb37779332b 2024-12-03T04:43:35,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/254c4dea9ddc462c93ff055b5a2cd263 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/254c4dea9ddc462c93ff055b5a2cd263 2024-12-03T04:43:35,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/f6d9e9cb68534b11b4a8e6340e2e9896 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/f6d9e9cb68534b11b4a8e6340e2e9896 2024-12-03T04:43:35,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9b5ac3e85694417aabd373b6c8740f0f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9b5ac3e85694417aabd373b6c8740f0f 2024-12-03T04:43:35,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/86c649189d77447e80f6903de27f9946 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/86c649189d77447e80f6903de27f9946 2024-12-03T04:43:35,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/b759460e60074868bd3765bdb09a55ea to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/b759460e60074868bd3765bdb09a55ea 2024-12-03T04:43:35,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/178f327cb875477fb6fdd283eff8b614 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/178f327cb875477fb6fdd283eff8b614 2024-12-03T04:43:35,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/561bfec4f7c24e888d95dc13d5091650 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/561bfec4f7c24e888d95dc13d5091650 2024-12-03T04:43:35,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/840029dc1a774ce4bd69de492f93cfdd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/b451529e47bb46f1b2e7b452dd5a8be7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/569f45e390c94bbd8900a3b207df3b95, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/6696b81cffdd4aee8e31e8659691b285, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88ea2bec57184cac8f58d6bd5d04f11a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/83af2fd015a044cfb2232148e6dba28e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/a1c658101b9c40bfbf6e1f69446bfe1f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/114e9c156c3648a3a9ebca3118a1e1a5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/f994d5ba872244b1a6f7a7ba5ad6d0c2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88c30c8ff7b143e4a543f599e34d1505, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/978f6de831af4027bb275d73dfde2908, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/89b6d62d86684a299393a833d712dd36, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/af0e645873c243febda1e21d42c5817d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/cfcc5dc9036b458a8cac3cc61a25d5af, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/908ed7dc2d6848d6ba7032b3c278bc21, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/03c47e83168c4deeb0c98c482dd9d7a0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/28850eaf88034aadb094644607dcb607, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/083c0bba7df043f8a2df21cd9ec64279, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/7df98600750e4a59bdc0214b916f1757, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/e48c3fac3d544d0d85823a42655cc1f7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/ca9d81f178f64c7596e4c6826007b2dc, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/89bb62d4688b4a4fa85a10c0a17be714, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/4efa219712034ea2af2ac252b7faaf2f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/408b9b7e37de4b46a3535d880ea126d3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/90bfb863394348c8b8f6dfbd0e8ff552, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/d09f82e2dd734cea9bb13850253b4ceb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/2939e9603a934c55b674f4065ac25f80, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/0b47ef060b8347928d33f68139378647, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/124f827cb3864155adb85dd553786463, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/8ced8e24743a4044b0bd63868fe3f12c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/224d40642828493a8c8d33db06b7c81d] to archive 2024-12-03T04:43:35,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:43:35,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/840029dc1a774ce4bd69de492f93cfdd to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/840029dc1a774ce4bd69de492f93cfdd 2024-12-03T04:43:35,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/b451529e47bb46f1b2e7b452dd5a8be7 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/b451529e47bb46f1b2e7b452dd5a8be7 2024-12-03T04:43:35,578 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/569f45e390c94bbd8900a3b207df3b95 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/569f45e390c94bbd8900a3b207df3b95 2024-12-03T04:43:35,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/6696b81cffdd4aee8e31e8659691b285 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/6696b81cffdd4aee8e31e8659691b285 2024-12-03T04:43:35,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88ea2bec57184cac8f58d6bd5d04f11a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88ea2bec57184cac8f58d6bd5d04f11a 2024-12-03T04:43:35,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/83af2fd015a044cfb2232148e6dba28e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/83af2fd015a044cfb2232148e6dba28e 2024-12-03T04:43:35,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/a1c658101b9c40bfbf6e1f69446bfe1f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/a1c658101b9c40bfbf6e1f69446bfe1f 2024-12-03T04:43:35,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/114e9c156c3648a3a9ebca3118a1e1a5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/114e9c156c3648a3a9ebca3118a1e1a5 2024-12-03T04:43:35,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/f994d5ba872244b1a6f7a7ba5ad6d0c2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/f994d5ba872244b1a6f7a7ba5ad6d0c2 2024-12-03T04:43:35,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88c30c8ff7b143e4a543f599e34d1505 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/88c30c8ff7b143e4a543f599e34d1505 2024-12-03T04:43:35,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/978f6de831af4027bb275d73dfde2908 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/978f6de831af4027bb275d73dfde2908 2024-12-03T04:43:35,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/89b6d62d86684a299393a833d712dd36 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/89b6d62d86684a299393a833d712dd36 2024-12-03T04:43:35,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/af0e645873c243febda1e21d42c5817d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/af0e645873c243febda1e21d42c5817d 2024-12-03T04:43:35,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/cfcc5dc9036b458a8cac3cc61a25d5af to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/cfcc5dc9036b458a8cac3cc61a25d5af 2024-12-03T04:43:35,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/908ed7dc2d6848d6ba7032b3c278bc21 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/908ed7dc2d6848d6ba7032b3c278bc21 2024-12-03T04:43:35,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/03c47e83168c4deeb0c98c482dd9d7a0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/03c47e83168c4deeb0c98c482dd9d7a0 2024-12-03T04:43:35,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/28850eaf88034aadb094644607dcb607 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/28850eaf88034aadb094644607dcb607 2024-12-03T04:43:35,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/083c0bba7df043f8a2df21cd9ec64279 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/083c0bba7df043f8a2df21cd9ec64279 2024-12-03T04:43:35,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/7df98600750e4a59bdc0214b916f1757 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/7df98600750e4a59bdc0214b916f1757 2024-12-03T04:43:35,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/e48c3fac3d544d0d85823a42655cc1f7 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/e48c3fac3d544d0d85823a42655cc1f7 2024-12-03T04:43:35,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/ca9d81f178f64c7596e4c6826007b2dc to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/ca9d81f178f64c7596e4c6826007b2dc 2024-12-03T04:43:35,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/89bb62d4688b4a4fa85a10c0a17be714 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/89bb62d4688b4a4fa85a10c0a17be714 2024-12-03T04:43:35,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/4efa219712034ea2af2ac252b7faaf2f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/4efa219712034ea2af2ac252b7faaf2f 2024-12-03T04:43:35,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/408b9b7e37de4b46a3535d880ea126d3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/408b9b7e37de4b46a3535d880ea126d3 2024-12-03T04:43:35,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/90bfb863394348c8b8f6dfbd0e8ff552 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/90bfb863394348c8b8f6dfbd0e8ff552 2024-12-03T04:43:35,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/d09f82e2dd734cea9bb13850253b4ceb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/d09f82e2dd734cea9bb13850253b4ceb 2024-12-03T04:43:35,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/2939e9603a934c55b674f4065ac25f80 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/2939e9603a934c55b674f4065ac25f80 2024-12-03T04:43:35,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/0b47ef060b8347928d33f68139378647 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/0b47ef060b8347928d33f68139378647 2024-12-03T04:43:35,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/124f827cb3864155adb85dd553786463 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/124f827cb3864155adb85dd553786463 2024-12-03T04:43:35,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/8ced8e24743a4044b0bd63868fe3f12c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/8ced8e24743a4044b0bd63868fe3f12c 2024-12-03T04:43:35,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/224d40642828493a8c8d33db06b7c81d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/224d40642828493a8c8d33db06b7c81d 2024-12-03T04:43:35,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a129253573a44b3eababd773271be6f9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f755edb8af9b411db220f4cd96b4fb17, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/435f7f0032af4c4697f5a932f0ccba6c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/70814b14c2674d83a525fbeae4181907, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/94c2922d75204c36a9f0bc090791b5ed, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cbb4c91d24de40418f01cbb14940baaa, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/334fdc3de91342abbd939dd5aad18d24, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c5968bdd3b4548fcacb6651a35b894b2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ea9baf092d07489ea5cd92abeb4a3b2b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cfd55afcd7134532b2e7d0000fbac48b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/221a0982816641b3931553958a2246fe, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/48f3e99a1c79453796baa8684b0918b8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ca07d48c5107412a8d68317fff21e017, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a20403c1d5ea4b68bf4f506b00ccdd4e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ece095c0bccf4fa4b8e5a4c2169519f3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/695564ad659043ff931ed3ea07e0c804, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/54ba73547c194c00bf50c8bd6b23fa3e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ee63d87def49436aa3da1e792816c8a8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/bdfe73bcf62b4b0ba2ac6582c134bd95, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a350682f84e44b91b9444e38560c554c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e5bcd9191ef1476a9bf14af7c6553a73, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/64b532c76a3d44d5b1a7480b4692ae3c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/09009144a7794f6e9d1738a0b2097380, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/57931cab71d24620ac01be9f4a3ac69d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/9252969e8511400f8242694a6c76e484, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c7d60cc816fc44ce908624875c243e51, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/0ce55feb876249598a6239b1b8cb76bb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/40ea5c7d461a4634854f152bf838da77, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f49897fcbefd4445bc57bf37c96ae4fb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e9aca328651649cb8943d240925e2624, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/5947c08c054449b6a98c2c2f109ad298] to archive 2024-12-03T04:43:35,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:43:35,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a129253573a44b3eababd773271be6f9 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a129253573a44b3eababd773271be6f9 2024-12-03T04:43:35,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f755edb8af9b411db220f4cd96b4fb17 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f755edb8af9b411db220f4cd96b4fb17 2024-12-03T04:43:35,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/435f7f0032af4c4697f5a932f0ccba6c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/435f7f0032af4c4697f5a932f0ccba6c 2024-12-03T04:43:35,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/70814b14c2674d83a525fbeae4181907 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/70814b14c2674d83a525fbeae4181907 2024-12-03T04:43:35,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/94c2922d75204c36a9f0bc090791b5ed to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/94c2922d75204c36a9f0bc090791b5ed 2024-12-03T04:43:35,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cbb4c91d24de40418f01cbb14940baaa to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cbb4c91d24de40418f01cbb14940baaa 2024-12-03T04:43:35,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/334fdc3de91342abbd939dd5aad18d24 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/334fdc3de91342abbd939dd5aad18d24 2024-12-03T04:43:35,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c5968bdd3b4548fcacb6651a35b894b2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c5968bdd3b4548fcacb6651a35b894b2 2024-12-03T04:43:35,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ea9baf092d07489ea5cd92abeb4a3b2b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ea9baf092d07489ea5cd92abeb4a3b2b 2024-12-03T04:43:35,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cfd55afcd7134532b2e7d0000fbac48b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/cfd55afcd7134532b2e7d0000fbac48b 2024-12-03T04:43:35,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/221a0982816641b3931553958a2246fe to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/221a0982816641b3931553958a2246fe 2024-12-03T04:43:35,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/48f3e99a1c79453796baa8684b0918b8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/48f3e99a1c79453796baa8684b0918b8 2024-12-03T04:43:35,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ca07d48c5107412a8d68317fff21e017 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ca07d48c5107412a8d68317fff21e017 2024-12-03T04:43:35,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a20403c1d5ea4b68bf4f506b00ccdd4e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a20403c1d5ea4b68bf4f506b00ccdd4e 2024-12-03T04:43:35,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ece095c0bccf4fa4b8e5a4c2169519f3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ece095c0bccf4fa4b8e5a4c2169519f3 2024-12-03T04:43:35,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/695564ad659043ff931ed3ea07e0c804 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/695564ad659043ff931ed3ea07e0c804 2024-12-03T04:43:35,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/54ba73547c194c00bf50c8bd6b23fa3e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/54ba73547c194c00bf50c8bd6b23fa3e 2024-12-03T04:43:35,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ee63d87def49436aa3da1e792816c8a8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/ee63d87def49436aa3da1e792816c8a8 2024-12-03T04:43:35,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/bdfe73bcf62b4b0ba2ac6582c134bd95 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/bdfe73bcf62b4b0ba2ac6582c134bd95 2024-12-03T04:43:35,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a350682f84e44b91b9444e38560c554c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/a350682f84e44b91b9444e38560c554c 2024-12-03T04:43:35,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e5bcd9191ef1476a9bf14af7c6553a73 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e5bcd9191ef1476a9bf14af7c6553a73 2024-12-03T04:43:35,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/64b532c76a3d44d5b1a7480b4692ae3c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/64b532c76a3d44d5b1a7480b4692ae3c 2024-12-03T04:43:35,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/09009144a7794f6e9d1738a0b2097380 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/09009144a7794f6e9d1738a0b2097380 2024-12-03T04:43:35,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/57931cab71d24620ac01be9f4a3ac69d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/57931cab71d24620ac01be9f4a3ac69d 2024-12-03T04:43:35,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/9252969e8511400f8242694a6c76e484 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/9252969e8511400f8242694a6c76e484 2024-12-03T04:43:35,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c7d60cc816fc44ce908624875c243e51 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/c7d60cc816fc44ce908624875c243e51 2024-12-03T04:43:35,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/0ce55feb876249598a6239b1b8cb76bb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/0ce55feb876249598a6239b1b8cb76bb 2024-12-03T04:43:35,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/40ea5c7d461a4634854f152bf838da77 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/40ea5c7d461a4634854f152bf838da77 2024-12-03T04:43:35,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f49897fcbefd4445bc57bf37c96ae4fb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/f49897fcbefd4445bc57bf37c96ae4fb 2024-12-03T04:43:35,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e9aca328651649cb8943d240925e2624 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/e9aca328651649cb8943d240925e2624 2024-12-03T04:43:35,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/5947c08c054449b6a98c2c2f109ad298 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/5947c08c054449b6a98c2c2f109ad298 2024-12-03T04:43:35,722 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/recovered.edits/586.seqid, newMaxSeqId=586, maxSeqId=1 2024-12-03T04:43:35,723 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. 2024-12-03T04:43:35,724 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 2444c4148e3a1c831bc6f992044b0533: Waiting for close lock at 1733201014899Running coprocessor pre-close hooks at 1733201014899Disabling compacts and flushes for region at 1733201014899Disabling writes for close at 1733201014899Obtaining lock to block concurrent updates at 1733201014899Preparing flush snapshotting stores in 2444c4148e3a1c831bc6f992044b0533 at 1733201014899Finished memstore snapshotting TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733201014901 (+2 ms)Flushing stores of TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533. at 1733201014901Flushing 2444c4148e3a1c831bc6f992044b0533/A: creating writer at 1733201014902 (+1 ms)Flushing 2444c4148e3a1c831bc6f992044b0533/A: appending metadata at 1733201014915 (+13 ms)Flushing 2444c4148e3a1c831bc6f992044b0533/A: closing flushed file at 1733201014915Flushing 2444c4148e3a1c831bc6f992044b0533/B: creating writer at 1733201015335 (+420 ms)Flushing 2444c4148e3a1c831bc6f992044b0533/B: appending metadata at 1733201015342 (+7 ms)Flushing 2444c4148e3a1c831bc6f992044b0533/B: closing flushed file at 1733201015343 (+1 ms)Flushing 2444c4148e3a1c831bc6f992044b0533/C: creating writer at 1733201015373 (+30 ms)Flushing 2444c4148e3a1c831bc6f992044b0533/C: appending metadata at 1733201015378 (+5 ms)Flushing 2444c4148e3a1c831bc6f992044b0533/C: closing flushed file at 1733201015378Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@288cb1ab: reopening flushed file at 1733201015394 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c8119fe: reopening flushed file at 1733201015401 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@479524b5: reopening flushed file at 1733201015409 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2444c4148e3a1c831bc6f992044b0533 in 518ms, sequenceid=583, compaction requested=true at 1733201015418 (+9 ms)Writing region close event to WAL at 1733201015717 (+299 ms)Running coprocessor post-close hooks at 1733201015723 (+6 ms)Closed at 1733201015723 2024-12-03T04:43:35,727 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:35,728 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=2444c4148e3a1c831bc6f992044b0533, regionState=CLOSED 2024-12-03T04:43:35,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:43:35,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T04:43:35,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 2444c4148e3a1c831bc6f992044b0533, server=32beccd76b35,43193,1733200986730 in 996 msec 2024-12-03T04:43:35,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-03T04:43:35,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2444c4148e3a1c831bc6f992044b0533, UNASSIGN in 1.0130 sec 2024-12-03T04:43:35,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-03T04:43:35,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0230 sec 2024-12-03T04:43:35,744 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201015744"}]},"ts":"1733201015744"} 2024-12-03T04:43:35,747 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T04:43:35,747 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T04:43:35,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0600 sec 2024-12-03T04:43:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T04:43:35,842 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:35,845 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T04:43:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:43:35,852 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:43:35,854 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:43:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T04:43:35,867 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:35,872 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/recovered.edits] 2024-12-03T04:43:35,876 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/3a95a4a16b5042d6a04346446b74f14c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/3a95a4a16b5042d6a04346446b74f14c 2024-12-03T04:43:35,878 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9d0b5f58111f48a794793c59be1ed171 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/9d0b5f58111f48a794793c59be1ed171 2024-12-03T04:43:35,879 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/d3bea4d15a8a48e4b8d8690896be640b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/A/d3bea4d15a8a48e4b8d8690896be640b 2024-12-03T04:43:35,885 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/0f2c2161a42b409ab17faa61f891e842 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/0f2c2161a42b409ab17faa61f891e842 2024-12-03T04:43:35,889 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/644562c452ef4d63b0287b58485b1531 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/644562c452ef4d63b0287b58485b1531 2024-12-03T04:43:35,891 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/6d7cf9e758054be0ae3500fbf3b4b2e0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/B/6d7cf9e758054be0ae3500fbf3b4b2e0 2024-12-03T04:43:35,894 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/204fe4e247834bcfa67bbd5b7912d9b8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/204fe4e247834bcfa67bbd5b7912d9b8 2024-12-03T04:43:35,896 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/7137e538d6eb4e3cb2a0fde2163fce5e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/7137e538d6eb4e3cb2a0fde2163fce5e 2024-12-03T04:43:35,901 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/8d7451ee4bbc471395f495d092c499f1 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/C/8d7451ee4bbc471395f495d092c499f1 2024-12-03T04:43:35,905 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/recovered.edits/586.seqid to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533/recovered.edits/586.seqid 2024-12-03T04:43:35,905 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/2444c4148e3a1c831bc6f992044b0533 2024-12-03T04:43:35,905 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T04:43:35,912 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:43:35,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-03T04:43:35,925 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T04:43:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T04:43:35,967 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T04:43:35,969 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:43:35,969 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T04:43:35,969 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733201015969"}]},"ts":"9223372036854775807"} 2024-12-03T04:43:35,978 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T04:43:35,978 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2444c4148e3a1c831bc6f992044b0533, NAME => 'TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T04:43:35,978 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T04:43:35,979 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733201015978"}]},"ts":"9223372036854775807"} 2024-12-03T04:43:35,982 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T04:43:35,983 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:43:35,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 138 msec 2024-12-03T04:43:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T04:43:36,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T04:43:36,172 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:36,190 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 209) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;32beccd76b35:43193-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1456175902_22 at /127.0.0.1:49952 [Waiting for operation #237] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1704177745_22 at /127.0.0.1:48862 [Waiting for operation #244] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1704177745_22 at /127.0.0.1:50140 [Waiting for operation #156] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/32beccd76b35:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=569 (was 298) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4905 (was 4661) - AvailableMemoryMB LEAK? - 2024-12-03T04:43:36,205 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=569, ProcessCount=11, AvailableMemoryMB=4904 2024-12-03T04:43:36,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T04:43:36,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:43:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T04:43:36,210 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T04:43:36,210 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:36,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-12-03T04:43:36,211 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T04:43:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T04:43:36,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741939_1115 (size=1061) 2024-12-03T04:43:36,224 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676 2024-12-03T04:43:36,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741940_1116 (size=53) 2024-12-03T04:43:36,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:43:36,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ad1a6acd12c06a97ea4048034fea205b, disabling compactions & flushes 2024-12-03T04:43:36,230 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:36,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:36,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. after waiting 0 ms 2024-12-03T04:43:36,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:36,230 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:36,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ad1a6acd12c06a97ea4048034fea205b: Waiting for close lock at 1733201016230Disabling compacts and flushes for region at 1733201016230Disabling writes for close at 1733201016230Writing region close event to WAL at 1733201016230Closed at 1733201016230 2024-12-03T04:43:36,231 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T04:43:36,232 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733201016231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733201016231"}]},"ts":"1733201016231"} 2024-12-03T04:43:36,234 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T04:43:36,234 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T04:43:36,235 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201016235"}]},"ts":"1733201016235"} 2024-12-03T04:43:36,236 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T04:43:36,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, ASSIGN}] 2024-12-03T04:43:36,238 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, ASSIGN 2024-12-03T04:43:36,239 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, ASSIGN; state=OFFLINE, location=32beccd76b35,43193,1733200986730; forceNewPlan=false, retain=false 2024-12-03T04:43:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T04:43:36,390 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ad1a6acd12c06a97ea4048034fea205b, regionState=OPENING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:43:36,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, ASSIGN because future has completed 2024-12-03T04:43:36,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:43:36,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T04:43:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T04:43:36,552 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:36,552 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:43:36,553 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,553 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:43:36,553 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,553 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,556 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,558 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:43:36,560 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad1a6acd12c06a97ea4048034fea205b columnFamilyName A 2024-12-03T04:43:36,561 DEBUG [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:36,561 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(327): Store=ad1a6acd12c06a97ea4048034fea205b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:36,561 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,563 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:43:36,563 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad1a6acd12c06a97ea4048034fea205b columnFamilyName B 2024-12-03T04:43:36,563 DEBUG [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:36,563 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(327): Store=ad1a6acd12c06a97ea4048034fea205b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:36,564 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,565 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:43:36,565 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad1a6acd12c06a97ea4048034fea205b columnFamilyName C 2024-12-03T04:43:36,565 DEBUG [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:36,566 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(327): Store=ad1a6acd12c06a97ea4048034fea205b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:36,566 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:36,566 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,567 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,567 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,569 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,569 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,569 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:43:36,571 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,574 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T04:43:36,575 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened ad1a6acd12c06a97ea4048034fea205b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75120870, jitterRate=0.11938819289207458}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:43:36,575 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:36,576 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for ad1a6acd12c06a97ea4048034fea205b: Running coprocessor pre-open hook at 1733201016554Writing region info on filesystem at 1733201016554Initializing all the Stores at 1733201016555 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201016555Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201016556 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201016556Cleaning up temporary data from old regions at 1733201016569 (+13 ms)Running coprocessor post-open hooks at 1733201016575 (+6 ms)Region opened successfully at 1733201016576 (+1 ms) 2024-12-03T04:43:36,577 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., pid=28, masterSystemTime=1733201016545 2024-12-03T04:43:36,580 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:36,580 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:36,582 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ad1a6acd12c06a97ea4048034fea205b, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:43:36,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:43:36,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-03T04:43:36,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 in 201 msec 2024-12-03T04:43:36,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-03T04:43:36,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, ASSIGN in 361 msec 2024-12-03T04:43:36,603 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T04:43:36,603 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201016603"}]},"ts":"1733201016603"} 2024-12-03T04:43:36,608 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T04:43:36,609 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T04:43:36,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 401 msec 2024-12-03T04:43:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T04:43:36,843 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:36,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49091041, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:36,845 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:36,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:36,848 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:36,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:36,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:36,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c3507b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:36,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:36,852 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:36,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a13e1f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:36,855 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:36,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:36,869 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T04:43:36,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,38397,1733200986071 2024-12-03T04:43:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6479eaa0 2024-12-03T04:43:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T04:43:36,871 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T04:43:36,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T04:43:36,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:43:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T04:43:36,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741941_1117 (size=1097) 2024-12-03T04:43:37,008 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T04:43:37,008 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T04:43:37,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T04:43:37,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, REOPEN/MOVE}] 2024-12-03T04:43:37,038 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, REOPEN/MOVE 2024-12-03T04:43:37,042 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=ad1a6acd12c06a97ea4048034fea205b, regionState=CLOSING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:43:37,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, REOPEN/MOVE because future has completed 2024-12-03T04:43:37,062 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T04:43:37,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:43:37,227 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,227 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T04:43:37,227 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing ad1a6acd12c06a97ea4048034fea205b, disabling compactions & flushes 2024-12-03T04:43:37,227 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:37,227 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:37,227 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. after waiting 0 ms 2024-12-03T04:43:37,227 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:37,241 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T04:43:37,242 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:37,242 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for ad1a6acd12c06a97ea4048034fea205b: Waiting for close lock at 1733201017227Running coprocessor pre-close hooks at 1733201017227Disabling compacts and flushes for region at 1733201017227Disabling writes for close at 1733201017227Writing region close event to WAL at 1733201017228 (+1 ms)Running coprocessor post-close hooks at 1733201017242 (+14 ms)Closed at 1733201017242 2024-12-03T04:43:37,242 WARN [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: ad1a6acd12c06a97ea4048034fea205b to self. 2024-12-03T04:43:37,245 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=ad1a6acd12c06a97ea4048034fea205b, regionState=CLOSED 2024-12-03T04:43:37,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:43:37,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T04:43:37,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 in 188 msec 2024-12-03T04:43:37,253 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, REOPEN/MOVE; state=CLOSED, location=32beccd76b35,43193,1733200986730; forceNewPlan=false, retain=true 2024-12-03T04:43:37,403 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=ad1a6acd12c06a97ea4048034fea205b, regionState=OPENING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:43:37,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, REOPEN/MOVE because future has completed 2024-12-03T04:43:37,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:43:37,563 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:37,563 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:43:37,564 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,564 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:43:37,564 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,564 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,570 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,570 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:43:37,577 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad1a6acd12c06a97ea4048034fea205b columnFamilyName A 2024-12-03T04:43:37,579 DEBUG [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:37,580 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(327): Store=ad1a6acd12c06a97ea4048034fea205b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:37,581 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,581 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:43:37,582 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad1a6acd12c06a97ea4048034fea205b columnFamilyName B 2024-12-03T04:43:37,582 DEBUG [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:37,582 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(327): Store=ad1a6acd12c06a97ea4048034fea205b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:37,582 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,583 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:43:37,583 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad1a6acd12c06a97ea4048034fea205b columnFamilyName C 2024-12-03T04:43:37,583 DEBUG [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:37,584 INFO [StoreOpener-ad1a6acd12c06a97ea4048034fea205b-1 {}] regionserver.HStore(327): Store=ad1a6acd12c06a97ea4048034fea205b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:43:37,584 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:37,584 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,585 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,586 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,588 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,588 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,588 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:43:37,590 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,591 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened ad1a6acd12c06a97ea4048034fea205b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69780489, jitterRate=0.039810314774513245}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:43:37,591 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,592 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for ad1a6acd12c06a97ea4048034fea205b: Running coprocessor pre-open hook at 1733201017564Writing region info on filesystem at 1733201017564Initializing all the Stores at 1733201017565 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201017565Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201017565Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201017565Cleaning up temporary data from old regions at 1733201017588 (+23 ms)Running coprocessor post-open hooks at 1733201017591 (+3 ms)Region opened successfully at 1733201017592 (+1 ms) 2024-12-03T04:43:37,593 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., pid=33, masterSystemTime=1733201017558 2024-12-03T04:43:37,595 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:37,595 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:37,596 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=ad1a6acd12c06a97ea4048034fea205b, regionState=OPEN, openSeqNum=5, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:43:37,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:43:37,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-12-03T04:43:37,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 in 193 msec 2024-12-03T04:43:37,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-03T04:43:37,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, REOPEN/MOVE in 566 msec 2024-12-03T04:43:37,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T04:43:37,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 586 msec 2024-12-03T04:43:37,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 722 msec 2024-12-03T04:43:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T04:43:37,613 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:37,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cce2794, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:37,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:37,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:37,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc91e4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:37,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:37,624 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5b1922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:37,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:37,630 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6096a572, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:37,633 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672b485d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:37,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:37,640 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c326d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:37,646 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d93fab4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:37,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:37,656 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:37,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:37,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,663 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:37,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:37,664 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:37,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:37,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:37,668 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd56d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:37,671 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292569fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:37,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:37,674 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3595572e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:37,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ec541b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:37,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:37,682 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29de59cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:37,684 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d0ad0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:37,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:37,686 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:37,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1f9e53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:37,688 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c72284e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:43:37,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:43:37,692 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:43:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:43:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:43:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b27ea61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:43:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:43:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:37,694 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:43:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2ecc06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:43:37,696 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:37,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:37,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:37,699 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:43:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-12-03T04:43:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T04:43:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:37,710 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:37,710 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:37,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:37,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:37,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:37,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:43:37,714 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:37,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:43:37,717 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:37,719 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:37,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:37,720 DEBUG [Thread-612 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:37,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:37,720 DEBUG [Thread-612 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:37,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:37,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:43:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:43:37,738 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:37,739 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:37,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:37,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:43:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:43:37,754 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:37,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:43:37,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:43:37,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:37,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:37,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:37,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:37,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:37,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,793 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:43:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:43:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:43:37,803 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:43:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:43:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T04:43:37,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T04:43:37,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:37,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:37,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:37,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:37,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:37,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1733201017824, max timestamp=1733201017878Num uniques 859; 2024-12-03T04:43:37,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-12-03T04:43:37,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1733201017824, max timestamp=1733201017878Num uniques 859; 2024-12-03T04:43:37,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-12-03T04:43:37,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1733201017824, max timestamp=1733201017878Num uniques 859; 2024-12-03T04:43:37,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-12-03T04:43:37,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120322bcb1f0aaff4031a72296f3ea77b9e9_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201017771/Put/seqid=0 2024-12-03T04:43:37,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741942_1118 (size=12154) 2024-12-03T04:43:37,939 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:37,945 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120322bcb1f0aaff4031a72296f3ea77b9e9_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120322bcb1f0aaff4031a72296f3ea77b9e9_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:37,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/262ec4bbf83b435aaf5ab2218740a7f5, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:37,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:37,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/262ec4bbf83b435aaf5ab2218740a7f5 is 175, key is test_row_0/A:col10/1733201017771/Put/seqid=0 2024-12-03T04:43:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201077935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:37,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201077937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201077937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201077944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:37,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201077971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:37,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:37,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:37,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T04:43:38,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:38,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:38,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:38,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:38,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T04:43:38,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741943_1119 (size=30951) 2024-12-03T04:43:38,061 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/262ec4bbf83b435aaf5ab2218740a7f5 2024-12-03T04:43:38,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201078082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201078082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201078083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201078092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201078102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/706b471ccd9145e4a7b5b975658986b2 is 50, key is test_row_0/B:col10/1733201017771/Put/seqid=0 2024-12-03T04:43:38,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741944_1120 (size=9657) 2024-12-03T04:43:38,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/706b471ccd9145e4a7b5b975658986b2 2024-12-03T04:43:38,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T04:43:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:38,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:38,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:38,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/39e391ed7bd34782ac7bee403943bdc2 is 50, key is test_row_0/C:col10/1733201017771/Put/seqid=0 2024-12-03T04:43:38,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741945_1121 (size=9657) 2024-12-03T04:43:38,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/39e391ed7bd34782ac7bee403943bdc2 2024-12-03T04:43:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/262ec4bbf83b435aaf5ab2218740a7f5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/262ec4bbf83b435aaf5ab2218740a7f5 2024-12-03T04:43:38,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/262ec4bbf83b435aaf5ab2218740a7f5, entries=150, sequenceid=14, filesize=30.2 K 2024-12-03T04:43:38,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/706b471ccd9145e4a7b5b975658986b2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/706b471ccd9145e4a7b5b975658986b2 2024-12-03T04:43:38,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201078314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201078314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201078315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201078315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/706b471ccd9145e4a7b5b975658986b2, entries=100, sequenceid=14, filesize=9.4 K 2024-12-03T04:43:38,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/39e391ed7bd34782ac7bee403943bdc2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/39e391ed7bd34782ac7bee403943bdc2 2024-12-03T04:43:38,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201078322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T04:43:38,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T04:43:38,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:38,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:38,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:38,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:38,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:38,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/39e391ed7bd34782ac7bee403943bdc2, entries=100, sequenceid=14, filesize=9.4 K 2024-12-03T04:43:38,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ad1a6acd12c06a97ea4048034fea205b in 604ms, sequenceid=14, compaction requested=false 2024-12-03T04:43:38,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:38,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T04:43:38,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:38,528 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=140.89 KB heapSize=372.09 KB 2024-12-03T04:43:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:38,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120387f12bc08c964cb3b37fd4cc679a171a_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201017932/Put/seqid=0 2024-12-03T04:43:38,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741946_1122 (size=12154) 2024-12-03T04:43:38,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:38,588 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120387f12bc08c964cb3b37fd4cc679a171a_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120387f12bc08c964cb3b37fd4cc679a171a_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:38,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/76c122cbfc844fafb1a9d806a944ac9c, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:38,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/76c122cbfc844fafb1a9d806a944ac9c is 175, key is test_row_0/A:col10/1733201017932/Put/seqid=0 2024-12-03T04:43:38,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741947_1123 (size=30955) 2024-12-03T04:43:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:38,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:38,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201078655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201078663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201078664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201078668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201078670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201078784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201078785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201078785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201078792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201078792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T04:43:39,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201078992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:39,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201078992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:39,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201079001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:39,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201079017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:39,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:39,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201079017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:39,030 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=45, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/76c122cbfc844fafb1a9d806a944ac9c 2024-12-03T04:43:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:39,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/8373ac837db746fd99d76ab0422650b3 is 50, key is test_row_0/B:col10/1733201017932/Put/seqid=0 2024-12-03T04:43:39,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741948_1124 (size=12001) 2024-12-03T04:43:39,118 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/8373ac837db746fd99d76ab0422650b3 2024-12-03T04:43:39,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/09fa721d3bb341619b6e77a5cfcb1f9c is 50, key is test_row_0/C:col10/1733201017932/Put/seqid=0 2024-12-03T04:43:39,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741949_1125 (size=12001) 2024-12-03T04:43:39,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201079322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:39,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201079322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:39,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201079325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:39,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:39,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:39,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201079340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:39,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201079342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:39,587 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/09fa721d3bb341619b6e77a5cfcb1f9c 2024-12-03T04:43:39,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/76c122cbfc844fafb1a9d806a944ac9c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/76c122cbfc844fafb1a9d806a944ac9c 2024-12-03T04:43:39,600 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/76c122cbfc844fafb1a9d806a944ac9c, entries=150, sequenceid=45, filesize=30.2 K 2024-12-03T04:43:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/8373ac837db746fd99d76ab0422650b3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/8373ac837db746fd99d76ab0422650b3 2024-12-03T04:43:39,607 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/8373ac837db746fd99d76ab0422650b3, entries=150, sequenceid=45, filesize=11.7 K 2024-12-03T04:43:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/09fa721d3bb341619b6e77a5cfcb1f9c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/09fa721d3bb341619b6e77a5cfcb1f9c 2024-12-03T04:43:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,633 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/09fa721d3bb341619b6e77a5cfcb1f9c, entries=150, sequenceid=45, filesize=11.7 K 2024-12-03T04:43:39,639 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~372.05 KB/380976, currentSize=46.96 KB/48090 for ad1a6acd12c06a97ea4048034fea205b in 1110ms, sequenceid=45, compaction requested=false 2024-12-03T04:43:39,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:39,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:39,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-03T04:43:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-12-03T04:43:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-03T04:43:39,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9300 sec 2024-12-03T04:43:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.9520 sec 2024-12-03T04:43:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T04:43:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,886 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-03T04:43:39,893 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:39,894 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:39,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T04:43:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:39,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T04:43:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:39,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cb0fbb2ee70a4888b6e83d81fa02a1a1_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201019915/Put/seqid=0 2024-12-03T04:43:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T04:43:40,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201079989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201080028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741950_1126 (size=14594) 2024-12-03T04:43:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,039 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201080034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,046 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cb0fbb2ee70a4888b6e83d81fa02a1a1_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cb0fbb2ee70a4888b6e83d81fa02a1a1_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:40,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T04:43:40,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:40,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201080038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201080038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,049 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/97dab83d37e74f9fa66d2fd344607ab3, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:40,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/97dab83d37e74f9fa66d2fd344607ab3 is 175, key is test_row_0/A:col10/1733201019915/Put/seqid=0 2024-12-03T04:43:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741951_1127 (size=39549) 2024-12-03T04:43:40,106 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/97dab83d37e74f9fa66d2fd344607ab3 2024-12-03T04:43:40,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b35b7e8074e14011aa7eada15840a773 is 50, key is test_row_0/B:col10/1733201019915/Put/seqid=0 2024-12-03T04:43:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201080141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201080142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201080156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201080156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201080164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741952_1128 (size=12001) 2024-12-03T04:43:40,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T04:43:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T04:43:40,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201080351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201080352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T04:43:40,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:40,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201080372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201080372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201080381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T04:43:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T04:43:40,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b35b7e8074e14011aa7eada15840a773 2024-12-03T04:43:40,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/eeed359fdd56470ba74ae3ad893cc58c is 50, key is test_row_0/C:col10/1733201019915/Put/seqid=0 2024-12-03T04:43:40,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201080661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741953_1129 (size=12001) 2024-12-03T04:43:40,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/eeed359fdd56470ba74ae3ad893cc58c 2024-12-03T04:43:40,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T04:43:40,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:40,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:40,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201080673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/97dab83d37e74f9fa66d2fd344607ab3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/97dab83d37e74f9fa66d2fd344607ab3 2024-12-03T04:43:40,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201080692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201080692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201080701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:40,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/97dab83d37e74f9fa66d2fd344607ab3, entries=200, sequenceid=59, filesize=38.6 K 2024-12-03T04:43:40,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b35b7e8074e14011aa7eada15840a773 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b35b7e8074e14011aa7eada15840a773 2024-12-03T04:43:40,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b35b7e8074e14011aa7eada15840a773, entries=150, sequenceid=59, filesize=11.7 K 2024-12-03T04:43:40,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/eeed359fdd56470ba74ae3ad893cc58c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/eeed359fdd56470ba74ae3ad893cc58c 2024-12-03T04:43:40,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/eeed359fdd56470ba74ae3ad893cc58c, entries=150, sequenceid=59, filesize=11.7 K 2024-12-03T04:43:40,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for ad1a6acd12c06a97ea4048034fea205b in 843ms, sequenceid=59, compaction requested=true 2024-12-03T04:43:40,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:40,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:40,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:40,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:40,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T04:43:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T04:43:40,759 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:40,760 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:40,760 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/A is initiating minor compaction (all files) 2024-12-03T04:43:40,760 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/A in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,760 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/262ec4bbf83b435aaf5ab2218740a7f5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/76c122cbfc844fafb1a9d806a944ac9c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/97dab83d37e74f9fa66d2fd344607ab3] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=99.1 K 2024-12-03T04:43:40,761 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,761 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:40,761 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/262ec4bbf83b435aaf5ab2218740a7f5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/76c122cbfc844fafb1a9d806a944ac9c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/97dab83d37e74f9fa66d2fd344607ab3] 2024-12-03T04:43:40,762 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 262ec4bbf83b435aaf5ab2218740a7f5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733201017753 2024-12-03T04:43:40,763 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:40,763 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/C is initiating minor compaction (all files) 2024-12-03T04:43:40,763 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/C in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,763 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/39e391ed7bd34782ac7bee403943bdc2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/09fa721d3bb341619b6e77a5cfcb1f9c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/eeed359fdd56470ba74ae3ad893cc58c] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=32.9 K 2024-12-03T04:43:40,763 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c122cbfc844fafb1a9d806a944ac9c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733201017932 2024-12-03T04:43:40,763 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 39e391ed7bd34782ac7bee403943bdc2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733201017753 2024-12-03T04:43:40,764 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 09fa721d3bb341619b6e77a5cfcb1f9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733201017932 2024-12-03T04:43:40,764 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97dab83d37e74f9fa66d2fd344607ab3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733201018663 2024-12-03T04:43:40,764 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting eeed359fdd56470ba74ae3ad893cc58c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733201018663 2024-12-03T04:43:40,780 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#C#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:40,781 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/cc7cd9ee9fa94cc1a8e5033412eda0bb is 50, key is test_row_0/C:col10/1733201019915/Put/seqid=0 2024-12-03T04:43:40,784 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:40,790 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412031b5737b6009d4e87a8d8c0de13869a54_ad1a6acd12c06a97ea4048034fea205b store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:40,794 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412031b5737b6009d4e87a8d8c0de13869a54_ad1a6acd12c06a97ea4048034fea205b, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:40,794 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031b5737b6009d4e87a8d8c0de13869a54_ad1a6acd12c06a97ea4048034fea205b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:40,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741954_1130 (size=12104) 2024-12-03T04:43:40,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T04:43:40,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:40,835 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T04:43:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:40,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741955_1131 (size=4469) 2024-12-03T04:43:40,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031663b8b95df542f9af5fd5acb9f11a6a_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201019979/Put/seqid=0 2024-12-03T04:43:40,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741956_1132 (size=12154) 2024-12-03T04:43:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T04:43:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:41,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:41,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201081216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201081218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201081220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201081222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201081227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,260 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/cc7cd9ee9fa94cc1a8e5033412eda0bb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/cc7cd9ee9fa94cc1a8e5033412eda0bb 2024-12-03T04:43:41,269 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/C of ad1a6acd12c06a97ea4048034fea205b into cc7cd9ee9fa94cc1a8e5033412eda0bb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:41,269 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:41,269 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/C, priority=13, startTime=1733201020758; duration=0sec 2024-12-03T04:43:41,269 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:41,270 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:C 2024-12-03T04:43:41,270 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:41,271 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:41,271 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/B is initiating minor compaction (all files) 2024-12-03T04:43:41,271 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/B in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:41,272 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/706b471ccd9145e4a7b5b975658986b2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/8373ac837db746fd99d76ab0422650b3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b35b7e8074e14011aa7eada15840a773] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=32.9 K 2024-12-03T04:43:41,274 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 706b471ccd9145e4a7b5b975658986b2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733201017753 2024-12-03T04:43:41,277 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 8373ac837db746fd99d76ab0422650b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733201017932 2024-12-03T04:43:41,278 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting b35b7e8074e14011aa7eada15840a773, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733201018663 2024-12-03T04:43:41,288 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:41,289 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/8373c9ea02d0411dbca86d6795834362 is 50, key is test_row_0/B:col10/1733201019915/Put/seqid=0 2024-12-03T04:43:41,303 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#A#compaction#112 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:41,303 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/45def0c6fb504877a1504f2c8e306bef is 175, key is test_row_0/A:col10/1733201019915/Put/seqid=0 2024-12-03T04:43:41,314 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T04:43:41,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201081334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201081334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741957_1133 (size=12104) 2024-12-03T04:43:41,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201081344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741958_1134 (size=31058) 2024-12-03T04:43:41,356 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/8373c9ea02d0411dbca86d6795834362 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/8373c9ea02d0411dbca86d6795834362 2024-12-03T04:43:41,367 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/B of ad1a6acd12c06a97ea4048034fea205b into 8373c9ea02d0411dbca86d6795834362(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:41,367 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:41,367 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/B, priority=13, startTime=1733201020758; duration=0sec 2024-12-03T04:43:41,367 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:41,367 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:B 2024-12-03T04:43:41,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201081355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:41,422 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031663b8b95df542f9af5fd5acb9f11a6a_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031663b8b95df542f9af5fd5acb9f11a6a_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:41,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/ddea4ad9bf52480b89524ef4a5b75e1f, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:41,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/ddea4ad9bf52480b89524ef4a5b75e1f is 175, key is test_row_0/A:col10/1733201019979/Put/seqid=0 2024-12-03T04:43:41,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741959_1135 (size=30955) 2024-12-03T04:43:41,507 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/ddea4ad9bf52480b89524ef4a5b75e1f 2024-12-03T04:43:41,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201081542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/de5fc0b3639e47d4bc72c8f04549d896 is 50, key is test_row_0/B:col10/1733201019979/Put/seqid=0 2024-12-03T04:43:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201081542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201081562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741960_1136 (size=12001) 2024-12-03T04:43:41,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,587 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/de5fc0b3639e47d4bc72c8f04549d896 2024-12-03T04:43:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201081572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/189ff607e20a4f2a9bf0f86750eda18d is 50, key is test_row_0/C:col10/1733201019979/Put/seqid=0 2024-12-03T04:43:41,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741961_1137 (size=12001) 2024-12-03T04:43:41,762 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/45def0c6fb504877a1504f2c8e306bef as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/45def0c6fb504877a1504f2c8e306bef 2024-12-03T04:43:41,768 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/A of ad1a6acd12c06a97ea4048034fea205b into 45def0c6fb504877a1504f2c8e306bef(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-12-03T04:43:41,769 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:41,769 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/A, priority=13, startTime=1733201020758; duration=1sec 2024-12-03T04:43:41,769 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:41,769 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:A 2024-12-03T04:43:41,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201081865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201081865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201081883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:41,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201081901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T04:43:42,112 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/189ff607e20a4f2a9bf0f86750eda18d 2024-12-03T04:43:42,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/ddea4ad9bf52480b89524ef4a5b75e1f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/ddea4ad9bf52480b89524ef4a5b75e1f 2024-12-03T04:43:42,143 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/ddea4ad9bf52480b89524ef4a5b75e1f, entries=150, sequenceid=78, filesize=30.2 K 2024-12-03T04:43:42,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/de5fc0b3639e47d4bc72c8f04549d896 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/de5fc0b3639e47d4bc72c8f04549d896 2024-12-03T04:43:42,185 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/de5fc0b3639e47d4bc72c8f04549d896, entries=150, sequenceid=78, filesize=11.7 K 2024-12-03T04:43:42,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/189ff607e20a4f2a9bf0f86750eda18d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/189ff607e20a4f2a9bf0f86750eda18d 2024-12-03T04:43:42,211 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/189ff607e20a4f2a9bf0f86750eda18d, entries=150, sequenceid=78, filesize=11.7 K 2024-12-03T04:43:42,222 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ad1a6acd12c06a97ea4048034fea205b in 1388ms, sequenceid=78, compaction requested=false 2024-12-03T04:43:42,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:42,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:42,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-03T04:43:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-03T04:43:42,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-03T04:43:42,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-12-03T04:43:42,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.3440 sec 2024-12-03T04:43:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:42,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T04:43:42,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:42,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:42,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:42,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:42,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:42,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:42,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ec4cef999794446c8a00f50025e5c06d_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201021214/Put/seqid=0 2024-12-03T04:43:42,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741962_1138 (size=12154) 2024-12-03T04:43:42,403 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:42,408 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ec4cef999794446c8a00f50025e5c06d_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ec4cef999794446c8a00f50025e5c06d_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:42,410 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/85a3d0063cfb468c8ab94252c6cef53e, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:42,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/85a3d0063cfb468c8ab94252c6cef53e is 175, key is test_row_0/A:col10/1733201021214/Put/seqid=0 2024-12-03T04:43:42,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201082404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201082409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201082412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201082412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201082426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741963_1139 (size=30955) 2024-12-03T04:43:42,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/85a3d0063cfb468c8ab94252c6cef53e 2024-12-03T04:43:42,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/952ce47bdb4a4f7abfa4b51ce555248a is 50, key is test_row_0/B:col10/1733201021214/Put/seqid=0 2024-12-03T04:43:42,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741964_1140 (size=12001) 2024-12-03T04:43:42,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201082532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201082533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201082533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201082551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201082753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201082753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201082761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201082782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:42,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/952ce47bdb4a4f7abfa4b51ce555248a 2024-12-03T04:43:42,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/f99303e4cddc4312832626d3ead492c1 is 50, key is test_row_0/C:col10/1733201021214/Put/seqid=0 2024-12-03T04:43:43,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741965_1141 (size=12001) 2024-12-03T04:43:43,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201083072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201083072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201083082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201083106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/f99303e4cddc4312832626d3ead492c1 2024-12-03T04:43:43,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201083434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,449 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in ad1a6acd12c06a97ea4048034fea205b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:43,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/85a3d0063cfb468c8ab94252c6cef53e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85a3d0063cfb468c8ab94252c6cef53e 2024-12-03T04:43:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85a3d0063cfb468c8ab94252c6cef53e, entries=150, sequenceid=95, filesize=30.2 K 2024-12-03T04:43:43,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/952ce47bdb4a4f7abfa4b51ce555248a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/952ce47bdb4a4f7abfa4b51ce555248a 2024-12-03T04:43:43,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/952ce47bdb4a4f7abfa4b51ce555248a, entries=150, sequenceid=95, filesize=11.7 K 2024-12-03T04:43:43,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/f99303e4cddc4312832626d3ead492c1 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f99303e4cddc4312832626d3ead492c1 2024-12-03T04:43:43,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f99303e4cddc4312832626d3ead492c1, entries=150, sequenceid=95, filesize=11.7 K 2024-12-03T04:43:43,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ad1a6acd12c06a97ea4048034fea205b in 1212ms, sequenceid=95, compaction requested=true 2024-12-03T04:43:43,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:43,477 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:43,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:43,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:43,478 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:43,478 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:43,479 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/A is initiating minor compaction (all files) 2024-12-03T04:43:43,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:43,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:43,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:43,479 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/A in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:43,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:43,479 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/45def0c6fb504877a1504f2c8e306bef, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/ddea4ad9bf52480b89524ef4a5b75e1f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85a3d0063cfb468c8ab94252c6cef53e] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=90.8 K 2024-12-03T04:43:43,479 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:43,479 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/45def0c6fb504877a1504f2c8e306bef, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/ddea4ad9bf52480b89524ef4a5b75e1f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85a3d0063cfb468c8ab94252c6cef53e] 2024-12-03T04:43:43,480 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45def0c6fb504877a1504f2c8e306bef, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733201018663 2024-12-03T04:43:43,480 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:43,480 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/B is initiating minor compaction (all files) 2024-12-03T04:43:43,480 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/B in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:43,480 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/8373c9ea02d0411dbca86d6795834362, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/de5fc0b3639e47d4bc72c8f04549d896, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/952ce47bdb4a4f7abfa4b51ce555248a] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=35.3 K 2024-12-03T04:43:43,481 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 8373c9ea02d0411dbca86d6795834362, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733201018663 2024-12-03T04:43:43,481 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddea4ad9bf52480b89524ef4a5b75e1f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733201019979 2024-12-03T04:43:43,482 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting de5fc0b3639e47d4bc72c8f04549d896, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733201019979 2024-12-03T04:43:43,482 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85a3d0063cfb468c8ab94252c6cef53e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733201021214 2024-12-03T04:43:43,483 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 952ce47bdb4a4f7abfa4b51ce555248a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733201021214 2024-12-03T04:43:43,498 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:43,508 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#B#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:43,508 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/60b699fd087940a589b9da953b4ef333 is 50, key is test_row_0/B:col10/1733201021214/Put/seqid=0 2024-12-03T04:43:43,514 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203f08357aa89674bd080b2737b8634fa83_ad1a6acd12c06a97ea4048034fea205b store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:43,519 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203f08357aa89674bd080b2737b8634fa83_ad1a6acd12c06a97ea4048034fea205b, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:43,520 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f08357aa89674bd080b2737b8634fa83_ad1a6acd12c06a97ea4048034fea205b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:43,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741966_1142 (size=12207) 2024-12-03T04:43:43,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T04:43:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:43,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:43,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:43,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:43,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:43,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:43,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:43,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741967_1143 (size=4469) 2024-12-03T04:43:43,615 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#A#compaction#120 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:43,615 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/2d44e3e32b80433088e96bd45485fd42 is 175, key is test_row_0/A:col10/1733201021214/Put/seqid=0 2024-12-03T04:43:43,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b0bd3ac03f944668beb71d78f3ba0e18_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201023584/Put/seqid=0 2024-12-03T04:43:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201083643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201083650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201083652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201083655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741968_1144 (size=31161) 2024-12-03T04:43:43,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741969_1145 (size=12154) 2024-12-03T04:43:43,722 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:43,735 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b0bd3ac03f944668beb71d78f3ba0e18_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b0bd3ac03f944668beb71d78f3ba0e18_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:43,736 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/142c6a6469074250abcaa220033b0df9, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:43,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/142c6a6469074250abcaa220033b0df9 is 175, key is test_row_0/A:col10/1733201023584/Put/seqid=0 2024-12-03T04:43:43,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741970_1146 (size=30955) 2024-12-03T04:43:43,757 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/142c6a6469074250abcaa220033b0df9 2024-12-03T04:43:43,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/49a9fc58d9a449ed9dce1f7e657567fe is 50, key is test_row_0/B:col10/1733201023584/Put/seqid=0 2024-12-03T04:43:43,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201083761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201083782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201083783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201083783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:43,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741971_1147 (size=12001) 2024-12-03T04:43:43,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/49a9fc58d9a449ed9dce1f7e657567fe 2024-12-03T04:43:43,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/58e1ff60307f4841a2e470df9ec24df3 is 50, key is test_row_0/C:col10/1733201023584/Put/seqid=0 2024-12-03T04:43:43,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741972_1148 (size=12001) 2024-12-03T04:43:43,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/58e1ff60307f4841a2e470df9ec24df3 2024-12-03T04:43:43,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/142c6a6469074250abcaa220033b0df9 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/142c6a6469074250abcaa220033b0df9 2024-12-03T04:43:43,988 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/60b699fd087940a589b9da953b4ef333 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/60b699fd087940a589b9da953b4ef333 2024-12-03T04:43:43,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/142c6a6469074250abcaa220033b0df9, entries=150, sequenceid=114, filesize=30.2 K 2024-12-03T04:43:43,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/49a9fc58d9a449ed9dce1f7e657567fe as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/49a9fc58d9a449ed9dce1f7e657567fe 2024-12-03T04:43:44,003 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/B of ad1a6acd12c06a97ea4048034fea205b into 60b699fd087940a589b9da953b4ef333(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:44,003 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:44,003 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/B, priority=13, startTime=1733201023478; duration=0sec 2024-12-03T04:43:44,004 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:44,004 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:B 2024-12-03T04:43:44,004 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:44,011 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:44,011 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/C is initiating minor compaction (all files) 2024-12-03T04:43:44,011 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/C in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:44,012 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/cc7cd9ee9fa94cc1a8e5033412eda0bb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/189ff607e20a4f2a9bf0f86750eda18d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f99303e4cddc4312832626d3ead492c1] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=35.3 K 2024-12-03T04:43:44,013 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting cc7cd9ee9fa94cc1a8e5033412eda0bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733201018663 2024-12-03T04:43:44,018 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 189ff607e20a4f2a9bf0f86750eda18d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733201019979 2024-12-03T04:43:44,021 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting f99303e4cddc4312832626d3ead492c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733201021214 2024-12-03T04:43:44,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/49a9fc58d9a449ed9dce1f7e657567fe, entries=150, sequenceid=114, filesize=11.7 K 2024-12-03T04:43:44,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/58e1ff60307f4841a2e470df9ec24df3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/58e1ff60307f4841a2e470df9ec24df3 2024-12-03T04:43:44,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201084002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201084003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201084003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201084003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/58e1ff60307f4841a2e470df9ec24df3, entries=150, sequenceid=114, filesize=11.7 K 2024-12-03T04:43:44,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ad1a6acd12c06a97ea4048034fea205b in 466ms, sequenceid=114, compaction requested=false 2024-12-03T04:43:44,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:44,055 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#C#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:44,056 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/a09803a4d063444ca2de68ac2509a600 is 50, key is test_row_0/C:col10/1733201021214/Put/seqid=0 2024-12-03T04:43:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T04:43:44,094 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:44,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-03T04:43:44,098 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T04:43:44,100 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:44,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:44,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741973_1149 (size=12207) 2024-12-03T04:43:44,119 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/2d44e3e32b80433088e96bd45485fd42 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2d44e3e32b80433088e96bd45485fd42 2024-12-03T04:43:44,130 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/a09803a4d063444ca2de68ac2509a600 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/a09803a4d063444ca2de68ac2509a600 2024-12-03T04:43:44,134 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/A of ad1a6acd12c06a97ea4048034fea205b into 2d44e3e32b80433088e96bd45485fd42(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:44,134 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:44,134 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/A, priority=13, startTime=1733201023477; duration=0sec 2024-12-03T04:43:44,134 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:44,134 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:A 2024-12-03T04:43:44,143 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/C of ad1a6acd12c06a97ea4048034fea205b into a09803a4d063444ca2de68ac2509a600(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:44,143 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:44,144 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/C, priority=13, startTime=1733201023479; duration=0sec 2024-12-03T04:43:44,144 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:44,144 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:C 2024-12-03T04:43:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T04:43:44,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T04:43:44,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:44,253 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T04:43:44,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:44,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:44,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:44,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:44,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:44,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:44,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b6946c65cb4844359b1f5c42289c89d6_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201023644/Put/seqid=0 2024-12-03T04:43:44,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:44,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741974_1150 (size=12154) 2024-12-03T04:43:44,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:44,369 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b6946c65cb4844359b1f5c42289c89d6_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b6946c65cb4844359b1f5c42289c89d6_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/aefacd6629da41c1bece474710853c4f, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:44,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/aefacd6629da41c1bece474710853c4f is 175, key is test_row_0/A:col10/1733201023644/Put/seqid=0 2024-12-03T04:43:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T04:43:44,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201084409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733201024377, max timestamp=1733201024430Num uniques 859; 2024-12-03T04:43:44,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:44,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741975_1151 (size=30955) 2024-12-03T04:43:44,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201024377, max timestamp=1733201024430Num uniques 859; 2024-12-03T04:43:44,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:44,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201024377, max timestamp=1733201024430Num uniques 859; 2024-12-03T04:43:44,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:44,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201084501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201084518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201084521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201084541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201084623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201084633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201084642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T04:43:44,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201084769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,838 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/aefacd6629da41c1bece474710853c4f 2024-12-03T04:43:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/6ff55132b2554947b660ba85070d3242 is 50, key is test_row_0/B:col10/1733201023644/Put/seqid=0 2024-12-03T04:43:44,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201084851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201084858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201084862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:44,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741976_1152 (size=12001) 2024-12-03T04:43:44,937 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/6ff55132b2554947b660ba85070d3242 2024-12-03T04:43:44,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/16f37375ae404d408a2a30beef51fae3 is 50, key is test_row_0/C:col10/1733201023644/Put/seqid=0 2024-12-03T04:43:45,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741977_1153 (size=12001) 2024-12-03T04:43:45,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201085092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:45,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201085182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:45,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201085183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:45,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201085183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:45,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T04:43:45,416 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/16f37375ae404d408a2a30beef51fae3 2024-12-03T04:43:45,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/aefacd6629da41c1bece474710853c4f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/aefacd6629da41c1bece474710853c4f 2024-12-03T04:43:45,441 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/aefacd6629da41c1bece474710853c4f, entries=150, sequenceid=130, filesize=30.2 K 2024-12-03T04:43:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/6ff55132b2554947b660ba85070d3242 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6ff55132b2554947b660ba85070d3242 2024-12-03T04:43:45,455 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6ff55132b2554947b660ba85070d3242, entries=150, sequenceid=130, filesize=11.7 K 2024-12-03T04:43:45,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/16f37375ae404d408a2a30beef51fae3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/16f37375ae404d408a2a30beef51fae3 2024-12-03T04:43:45,480 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/16f37375ae404d408a2a30beef51fae3, entries=150, sequenceid=130, filesize=11.7 K 2024-12-03T04:43:45,481 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=120.76 KB/123660 for ad1a6acd12c06a97ea4048034fea205b in 1228ms, sequenceid=130, compaction requested=true 2024-12-03T04:43:45,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:45,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:45,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-03T04:43:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-03T04:43:45,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-03T04:43:45,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3830 sec 2024-12-03T04:43:45,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.3910 sec 2024-12-03T04:43:45,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-03T04:43:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:45,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:45,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033f4d3edce65e42e386844c92b0bb55b4_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201025491/Put/seqid=0 2024-12-03T04:43:45,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741978_1154 (size=14794) 2024-12-03T04:43:45,597 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:45,606 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033f4d3edce65e42e386844c92b0bb55b4_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033f4d3edce65e42e386844c92b0bb55b4_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:45,608 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/d50f2c485add4296af880f3eda077200, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:45,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/d50f2c485add4296af880f3eda077200 is 175, key is test_row_0/A:col10/1733201025491/Put/seqid=0 2024-12-03T04:43:45,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201085621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:45,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201085629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:45,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:45,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741979_1155 (size=39749) 2024-12-03T04:43:45,654 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/d50f2c485add4296af880f3eda077200 2024-12-03T04:43:45,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/20ec4163d88b43b0a68f1739cf0b57c5 is 50, key is test_row_0/B:col10/1733201025491/Put/seqid=0 2024-12-03T04:43:45,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201085702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:45,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741980_1156 (size=12151) 2024-12-03T04:43:45,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/20ec4163d88b43b0a68f1739cf0b57c5 2024-12-03T04:43:45,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201085712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:45,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201085712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:45,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/d46e47b2bab14f3abe97bda0807b2906 is 50, key is test_row_0/C:col10/1733201025491/Put/seqid=0 2024-12-03T04:43:45,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201085752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:45,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741981_1157 (size=12151) 2024-12-03T04:43:45,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/d46e47b2bab14f3abe97bda0807b2906 2024-12-03T04:43:45,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/d50f2c485add4296af880f3eda077200 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d50f2c485add4296af880f3eda077200 2024-12-03T04:43:45,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d50f2c485add4296af880f3eda077200, entries=200, sequenceid=165, filesize=38.8 K 2024-12-03T04:43:45,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/20ec4163d88b43b0a68f1739cf0b57c5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/20ec4163d88b43b0a68f1739cf0b57c5 2024-12-03T04:43:45,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/20ec4163d88b43b0a68f1739cf0b57c5, entries=150, sequenceid=165, filesize=11.9 K 2024-12-03T04:43:45,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/d46e47b2bab14f3abe97bda0807b2906 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/d46e47b2bab14f3abe97bda0807b2906 2024-12-03T04:43:45,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/d46e47b2bab14f3abe97bda0807b2906, entries=150, sequenceid=165, filesize=11.9 K 2024-12-03T04:43:45,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ad1a6acd12c06a97ea4048034fea205b in 345ms, sequenceid=165, compaction requested=true 2024-12-03T04:43:45,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:45,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:45,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:45,840 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:45,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:45,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:45,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:45,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T04:43:45,840 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:45,842 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:45,842 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/B is initiating minor compaction (all files) 2024-12-03T04:43:45,842 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/B in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:45,842 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/60b699fd087940a589b9da953b4ef333, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/49a9fc58d9a449ed9dce1f7e657567fe, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6ff55132b2554947b660ba85070d3242, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/20ec4163d88b43b0a68f1739cf0b57c5] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=47.2 K 2024-12-03T04:43:45,843 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:45,843 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/A is initiating minor compaction (all files) 2024-12-03T04:43:45,843 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/A in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:45,843 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2d44e3e32b80433088e96bd45485fd42, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/142c6a6469074250abcaa220033b0df9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/aefacd6629da41c1bece474710853c4f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d50f2c485add4296af880f3eda077200] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=129.7 K 2024-12-03T04:43:45,843 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:45,843 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2d44e3e32b80433088e96bd45485fd42, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/142c6a6469074250abcaa220033b0df9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/aefacd6629da41c1bece474710853c4f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d50f2c485add4296af880f3eda077200] 2024-12-03T04:43:45,844 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60b699fd087940a589b9da953b4ef333, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733201021214 2024-12-03T04:43:45,847 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d44e3e32b80433088e96bd45485fd42, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733201021214 2024-12-03T04:43:45,848 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49a9fc58d9a449ed9dce1f7e657567fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733201022409 2024-12-03T04:43:45,848 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 142c6a6469074250abcaa220033b0df9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733201022409 2024-12-03T04:43:45,848 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ff55132b2554947b660ba85070d3242, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733201023644 2024-12-03T04:43:45,849 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting aefacd6629da41c1bece474710853c4f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733201023644 2024-12-03T04:43:45,849 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20ec4163d88b43b0a68f1739cf0b57c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733201024483 2024-12-03T04:43:45,850 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting d50f2c485add4296af880f3eda077200, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733201024483 2024-12-03T04:43:45,872 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:45,873 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/ec4d2c1b3fe2431b8bcf0ff4fab1752d is 50, key is test_row_0/B:col10/1733201025491/Put/seqid=0 2024-12-03T04:43:45,888 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:45,902 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203a3d14a82233243f4a4e96b4b8a098cfa_ad1a6acd12c06a97ea4048034fea205b store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:45,905 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203a3d14a82233243f4a4e96b4b8a098cfa_ad1a6acd12c06a97ea4048034fea205b, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:45,906 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a3d14a82233243f4a4e96b4b8a098cfa_ad1a6acd12c06a97ea4048034fea205b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:45,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741982_1158 (size=12493) 2024-12-03T04:43:45,977 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/ec4d2c1b3fe2431b8bcf0ff4fab1752d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/ec4d2c1b3fe2431b8bcf0ff4fab1752d 2024-12-03T04:43:45,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:43:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:45,984 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/B of ad1a6acd12c06a97ea4048034fea205b into ec4d2c1b3fe2431b8bcf0ff4fab1752d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:45,984 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:45,984 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/B, priority=12, startTime=1733201025840; duration=0sec 2024-12-03T04:43:45,984 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:45,984 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:B 2024-12-03T04:43:45,984 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:45,986 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:45,987 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/C is initiating minor compaction (all files) 2024-12-03T04:43:45,987 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/C in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:45,987 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/a09803a4d063444ca2de68ac2509a600, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/58e1ff60307f4841a2e470df9ec24df3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/16f37375ae404d408a2a30beef51fae3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/d46e47b2bab14f3abe97bda0807b2906] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=47.2 K 2024-12-03T04:43:45,987 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting a09803a4d063444ca2de68ac2509a600, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733201021214 2024-12-03T04:43:45,989 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58e1ff60307f4841a2e470df9ec24df3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733201022409 2024-12-03T04:43:45,989 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16f37375ae404d408a2a30beef51fae3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733201023644 2024-12-03T04:43:45,990 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting d46e47b2bab14f3abe97bda0807b2906, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733201024483 2024-12-03T04:43:46,006 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#C#compaction#134 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:46,007 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/e16bb56e376947bd90f3007d40d32358 is 50, key is test_row_0/C:col10/1733201025491/Put/seqid=0 2024-12-03T04:43:46,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037c80698c279943b08a1d45b85c6b420a_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201025601/Put/seqid=0 2024-12-03T04:43:46,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741983_1159 (size=4469) 2024-12-03T04:43:46,043 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#A#compaction#133 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:46,044 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/68bc2f877434440e90cd72a76dd9dd64 is 175, key is test_row_0/A:col10/1733201025491/Put/seqid=0 2024-12-03T04:43:46,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741985_1161 (size=14794) 2024-12-03T04:43:46,113 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:46,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741984_1160 (size=12493) 2024-12-03T04:43:46,121 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037c80698c279943b08a1d45b85c6b420a_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037c80698c279943b08a1d45b85c6b420a_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:46,125 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/6ba7a23f55de4618b866ea907292ae92, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:46,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/6ba7a23f55de4618b866ea907292ae92 is 175, key is test_row_0/A:col10/1733201025601/Put/seqid=0 2024-12-03T04:43:46,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741986_1162 (size=31447) 2024-12-03T04:43:46,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741987_1163 (size=39749) 2024-12-03T04:43:46,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201086211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T04:43:46,257 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:46,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-03T04:43:46,259 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T04:43:46,260 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:46,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:46,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201086332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T04:43:46,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T04:43:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:46,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:46,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T04:43:46,470 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T04:43:46,524 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/e16bb56e376947bd90f3007d40d32358 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/e16bb56e376947bd90f3007d40d32358 2024-12-03T04:43:46,538 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/C of ad1a6acd12c06a97ea4048034fea205b into e16bb56e376947bd90f3007d40d32358(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:46,539 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:46,539 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/C, priority=12, startTime=1733201025840; duration=0sec 2024-12-03T04:43:46,539 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:46,539 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:C 2024-12-03T04:43:46,546 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/68bc2f877434440e90cd72a76dd9dd64 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/68bc2f877434440e90cd72a76dd9dd64 2024-12-03T04:43:46,552 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/A of ad1a6acd12c06a97ea4048034fea205b into 68bc2f877434440e90cd72a76dd9dd64(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:46,552 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:46,552 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/A, priority=12, startTime=1733201025840; duration=0sec 2024-12-03T04:43:46,552 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:46,552 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:A 2024-12-03T04:43:46,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201086541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:46,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:46,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T04:43:46,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:46,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:46,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:46,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:46,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T04:43:46,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/6ba7a23f55de4618b866ea907292ae92 2024-12-03T04:43:46,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/071bb2e4f8a44e70a14d15b3657c85bc is 50, key is test_row_0/B:col10/1733201025601/Put/seqid=0 2024-12-03T04:43:46,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201086642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:46,647 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in ad1a6acd12c06a97ea4048034fea205b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:46,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741988_1164 (size=12151) 2024-12-03T04:43:46,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/071bb2e4f8a44e70a14d15b3657c85bc 2024-12-03T04:43:46,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/ae93a879701a44ebab495b71fdd118fd is 50, key is test_row_0/C:col10/1733201025601/Put/seqid=0 2024-12-03T04:43:46,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:46,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T04:43:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201086722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:46,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:46,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:46,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:46,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:46,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:46,725 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in ad1a6acd12c06a97ea4048034fea205b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:46,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201086732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:46,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201086732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:46,735 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in ad1a6acd12c06a97ea4048034fea205b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:46,736 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in ad1a6acd12c06a97ea4048034fea205b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:46,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741989_1165 (size=12151) 2024-12-03T04:43:46,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201086872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T04:43:46,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T04:43:46,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:46,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:46,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:46,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:46,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:47,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T04:43:47,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:47,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:47,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:47,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:47,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:47,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:47,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/ae93a879701a44ebab495b71fdd118fd 2024-12-03T04:43:47,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/6ba7a23f55de4618b866ea907292ae92 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/6ba7a23f55de4618b866ea907292ae92 2024-12-03T04:43:47,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/6ba7a23f55de4618b866ea907292ae92, entries=200, sequenceid=177, filesize=38.8 K 2024-12-03T04:43:47,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/071bb2e4f8a44e70a14d15b3657c85bc as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/071bb2e4f8a44e70a14d15b3657c85bc 2024-12-03T04:43:47,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T04:43:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:47,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:47,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/071bb2e4f8a44e70a14d15b3657c85bc, entries=150, sequenceid=177, filesize=11.9 K 2024-12-03T04:43:47,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/ae93a879701a44ebab495b71fdd118fd as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ae93a879701a44ebab495b71fdd118fd 2024-12-03T04:43:47,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ae93a879701a44ebab495b71fdd118fd, entries=150, sequenceid=177, filesize=11.9 K 2024-12-03T04:43:47,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ad1a6acd12c06a97ea4048034fea205b in 1285ms, sequenceid=177, compaction requested=false 2024-12-03T04:43:47,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:47,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T04:43:47,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:47,353 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:43:47,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:47,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:47,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:47,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:47,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:47,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:47,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T04:43:47,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038df5e1d224624b4693ab56042d835fd1_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201026137/Put/seqid=0 2024-12-03T04:43:47,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741990_1166 (size=12304) 2024-12-03T04:43:47,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:47,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201087492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:47,513 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038df5e1d224624b4693ab56042d835fd1_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038df5e1d224624b4693ab56042d835fd1_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:47,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/e5b0654e3aa24f8c9588e0969d86e0ae, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:47,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/e5b0654e3aa24f8c9588e0969d86e0ae is 175, key is test_row_0/A:col10/1733201026137/Put/seqid=0 2024-12-03T04:43:47,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741991_1167 (size=31105) 2024-12-03T04:43:47,565 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/e5b0654e3aa24f8c9588e0969d86e0ae 2024-12-03T04:43:47,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b098e10bc92e49d9a37e9b14a20a01f4 is 50, key is test_row_0/B:col10/1733201026137/Put/seqid=0 2024-12-03T04:43:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201087612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:47,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741992_1168 (size=12151) 2024-12-03T04:43:47,637 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b098e10bc92e49d9a37e9b14a20a01f4 2024-12-03T04:43:47,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/891b828b84ea4e1e8cdd687a868e7f42 is 50, key is test_row_0/C:col10/1733201026137/Put/seqid=0 2024-12-03T04:43:47,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741993_1169 (size=12151) 2024-12-03T04:43:47,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201087822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:47,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:47,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:47,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:48,110 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/891b828b84ea4e1e8cdd687a868e7f42 2024-12-03T04:43:48,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/e5b0654e3aa24f8c9588e0969d86e0ae as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/e5b0654e3aa24f8c9588e0969d86e0ae 2024-12-03T04:43:48,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201088142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:48,146 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/e5b0654e3aa24f8c9588e0969d86e0ae, entries=150, sequenceid=200, filesize=30.4 K 2024-12-03T04:43:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:48,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b098e10bc92e49d9a37e9b14a20a01f4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b098e10bc92e49d9a37e9b14a20a01f4 2024-12-03T04:43:48,158 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b098e10bc92e49d9a37e9b14a20a01f4, entries=150, sequenceid=200, filesize=11.9 K 2024-12-03T04:43:48,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/891b828b84ea4e1e8cdd687a868e7f42 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/891b828b84ea4e1e8cdd687a868e7f42 2024-12-03T04:43:48,185 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/891b828b84ea4e1e8cdd687a868e7f42, entries=150, sequenceid=200, filesize=11.9 K 2024-12-03T04:43:48,208 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ad1a6acd12c06a97ea4048034fea205b in 854ms, sequenceid=200, compaction requested=true 2024-12-03T04:43:48,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:48,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:48,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-03T04:43:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-03T04:43:48,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-03T04:43:48,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9550 sec 2024-12-03T04:43:48,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.9620 sec 2024-12-03T04:43:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T04:43:48,402 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:48,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-03T04:43:48,407 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T04:43:48,408 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:48,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T04:43:48,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T04:43:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:48,561 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:43:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:48,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b8c6ab4610e141159b6c0c60f6004fb4_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201027484/Put/seqid=0 2024-12-03T04:43:48,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741994_1170 (size=12304) 2024-12-03T04:43:48,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:48,609 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b8c6ab4610e141159b6c0c60f6004fb4_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b8c6ab4610e141159b6c0c60f6004fb4_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:48,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/85b48c3d9a7f40eba1ba945871d1adfe, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:48,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/85b48c3d9a7f40eba1ba945871d1adfe is 175, key is test_row_0/A:col10/1733201027484/Put/seqid=0 2024-12-03T04:43:48,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741995_1171 (size=31105) 2024-12-03T04:43:48,617 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/85b48c3d9a7f40eba1ba945871d1adfe 2024-12-03T04:43:48,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/1e2ce25292f84c4f913d21593d66f5e1 is 50, key is test_row_0/B:col10/1733201027484/Put/seqid=0 2024-12-03T04:43:48,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741996_1172 (size=12151) 2024-12-03T04:43:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:48,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T04:43:48,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201088739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:48,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201088741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:48,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:48,750 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in ad1a6acd12c06a97ea4048034fea205b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:48,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201088750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:48,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201088756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:48,764 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in ad1a6acd12c06a97ea4048034fea205b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:48,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201088761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:48,767 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in ad1a6acd12c06a97ea4048034fea205b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:48,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:48,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201088861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:48,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:48,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201088863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:48,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:48,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T04:43:49,045 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/1e2ce25292f84c4f913d21593d66f5e1 2024-12-03T04:43:49,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/c52dc0c9112b481a9a1125afcfeee47b is 50, key is test_row_0/C:col10/1733201027484/Put/seqid=0 2024-12-03T04:43:49,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201089072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:49,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201089072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:49,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741997_1173 (size=12151) 2024-12-03T04:43:49,113 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/c52dc0c9112b481a9a1125afcfeee47b 2024-12-03T04:43:49,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/85b48c3d9a7f40eba1ba945871d1adfe as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85b48c3d9a7f40eba1ba945871d1adfe 2024-12-03T04:43:49,174 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85b48c3d9a7f40eba1ba945871d1adfe, entries=150, sequenceid=212, filesize=30.4 K 2024-12-03T04:43:49,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/1e2ce25292f84c4f913d21593d66f5e1 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/1e2ce25292f84c4f913d21593d66f5e1 2024-12-03T04:43:49,194 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/1e2ce25292f84c4f913d21593d66f5e1, entries=150, sequenceid=212, filesize=11.9 K 2024-12-03T04:43:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/c52dc0c9112b481a9a1125afcfeee47b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/c52dc0c9112b481a9a1125afcfeee47b 2024-12-03T04:43:49,218 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/c52dc0c9112b481a9a1125afcfeee47b, entries=150, sequenceid=212, filesize=11.9 K 2024-12-03T04:43:49,222 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ad1a6acd12c06a97ea4048034fea205b in 661ms, sequenceid=212, compaction requested=true 2024-12-03T04:43:49,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:49,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:49,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-03T04:43:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-03T04:43:49,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-03T04:43:49,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 816 msec 2024-12-03T04:43:49,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 825 msec 2024-12-03T04:43:49,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201028702, max timestamp=1733201029397Num uniques 859; 2024-12-03T04:43:49,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:49,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201028702, max timestamp=1733201029397Num uniques 859; 2024-12-03T04:43:49,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:49,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-03T04:43:49,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:49,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:49,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:49,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:49,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:49,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:49,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:49,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d3e064e0403e4980afc9097bbdc507d3_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201028735/Put/seqid=0 2024-12-03T04:43:49,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741998_1174 (size=12304) 2024-12-03T04:43:49,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733201029481, max timestamp=1733201029532Num uniques 859; 2024-12-03T04:43:49,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:43:49,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733201029481, max timestamp=1733201029532Num uniques 859; 2024-12-03T04:43:49,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:43:49,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201029502, max timestamp=1733201029532Num uniques 859; 2024-12-03T04:43:49,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T04:43:49,542 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:49,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-03T04:43:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T04:43:49,546 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:49,547 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:49,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201089630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:49,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201089631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:49,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T04:43:49,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T04:43:49,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:49,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:49,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201089742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:49,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:49,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201089742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T04:43:49,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T04:43:49,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:49,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:49,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:49,925 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:49,938 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d3e064e0403e4980afc9097bbdc507d3_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d3e064e0403e4980afc9097bbdc507d3_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:49,940 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/18993899492d480da5578ed72052f3fd, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:49,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/18993899492d480da5578ed72052f3fd is 175, key is test_row_0/A:col10/1733201028735/Put/seqid=0 2024-12-03T04:43:49,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741999_1175 (size=31105) 2024-12-03T04:43:49,958 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/18993899492d480da5578ed72052f3fd 2024-12-03T04:43:49,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201089952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201089951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:49,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b78cc94adf3e4ad79dbc4b9b48b3dc33 is 50, key is test_row_0/B:col10/1733201028735/Put/seqid=0 2024-12-03T04:43:49,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742000_1176 (size=12151) 2024-12-03T04:43:49,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b78cc94adf3e4ad79dbc4b9b48b3dc33 2024-12-03T04:43:50,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/af7553a201684facac4e8115211c3649 is 50, key is test_row_0/C:col10/1733201028735/Put/seqid=0 2024-12-03T04:43:50,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742001_1177 (size=12151) 2024-12-03T04:43:50,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T04:43:50,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:50,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:50,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:50,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:50,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T04:43:50,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T04:43:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:50,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:50,182 INFO [master/32beccd76b35:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T04:43:50,182 INFO [master/32beccd76b35:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T04:43:50,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201090271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201090271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:50,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T04:43:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:50,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:50,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:50,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/af7553a201684facac4e8115211c3649 2024-12-03T04:43:50,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/18993899492d480da5578ed72052f3fd as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/18993899492d480da5578ed72052f3fd 2024-12-03T04:43:50,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/18993899492d480da5578ed72052f3fd, entries=150, sequenceid=233, filesize=30.4 K 2024-12-03T04:43:50,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b78cc94adf3e4ad79dbc4b9b48b3dc33 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b78cc94adf3e4ad79dbc4b9b48b3dc33 2024-12-03T04:43:50,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b78cc94adf3e4ad79dbc4b9b48b3dc33, entries=150, sequenceid=233, filesize=11.9 K 2024-12-03T04:43:50,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/af7553a201684facac4e8115211c3649 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/af7553a201684facac4e8115211c3649 2024-12-03T04:43:50,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/af7553a201684facac4e8115211c3649, entries=150, sequenceid=233, filesize=11.9 K 2024-12-03T04:43:50,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for ad1a6acd12c06a97ea4048034fea205b in 1071ms, sequenceid=233, compaction requested=true 2024-12-03T04:43:50,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:50,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:50,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:50,479 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T04:43:50,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:50,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:50,479 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T04:43:50,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:50,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:50,481 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61097 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T04:43:50,481 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164511 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T04:43:50,481 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/B is initiating minor compaction (all files) 2024-12-03T04:43:50,481 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/A is initiating minor compaction (all files) 2024-12-03T04:43:50,481 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/B in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:50,481 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/A in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:50,481 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/ec4d2c1b3fe2431b8bcf0ff4fab1752d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/071bb2e4f8a44e70a14d15b3657c85bc, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b098e10bc92e49d9a37e9b14a20a01f4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/1e2ce25292f84c4f913d21593d66f5e1, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b78cc94adf3e4ad79dbc4b9b48b3dc33] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=59.7 K 2024-12-03T04:43:50,482 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/68bc2f877434440e90cd72a76dd9dd64, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/6ba7a23f55de4618b866ea907292ae92, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/e5b0654e3aa24f8c9588e0969d86e0ae, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85b48c3d9a7f40eba1ba945871d1adfe, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/18993899492d480da5578ed72052f3fd] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=160.7 K 2024-12-03T04:43:50,482 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:50,482 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/68bc2f877434440e90cd72a76dd9dd64, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/6ba7a23f55de4618b866ea907292ae92, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/e5b0654e3aa24f8c9588e0969d86e0ae, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85b48c3d9a7f40eba1ba945871d1adfe, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/18993899492d480da5578ed72052f3fd] 2024-12-03T04:43:50,482 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ec4d2c1b3fe2431b8bcf0ff4fab1752d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733201024483 2024-12-03T04:43:50,482 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68bc2f877434440e90cd72a76dd9dd64, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733201024483 2024-12-03T04:43:50,483 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 071bb2e4f8a44e70a14d15b3657c85bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733201025510 2024-12-03T04:43:50,483 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ba7a23f55de4618b866ea907292ae92, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733201025510 2024-12-03T04:43:50,483 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting b098e10bc92e49d9a37e9b14a20a01f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733201026137 2024-12-03T04:43:50,483 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5b0654e3aa24f8c9588e0969d86e0ae, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733201026137 2024-12-03T04:43:50,484 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e2ce25292f84c4f913d21593d66f5e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733201027446 2024-12-03T04:43:50,484 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85b48c3d9a7f40eba1ba945871d1adfe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733201027446 2024-12-03T04:43:50,485 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting b78cc94adf3e4ad79dbc4b9b48b3dc33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733201028735 2024-12-03T04:43:50,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T04:43:50,485 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18993899492d480da5578ed72052f3fd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733201028735 2024-12-03T04:43:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:50,485 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-03T04:43:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:50,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:50,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:50,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:50,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:50,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:50,516 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:50,517 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/62da020adf344a50ab911ce8e7e3af05 is 50, key is test_row_0/B:col10/1733201028735/Put/seqid=0 2024-12-03T04:43:50,522 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:50,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f3eb36c298af4ae59e48788e89f473c3_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201029614/Put/seqid=0 2024-12-03T04:43:50,572 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033c6cfdf2c20e414f824ef5d705a30c30_ad1a6acd12c06a97ea4048034fea205b store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:50,575 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033c6cfdf2c20e414f824ef5d705a30c30_ad1a6acd12c06a97ea4048034fea205b, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:50,575 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033c6cfdf2c20e414f824ef5d705a30c30_ad1a6acd12c06a97ea4048034fea205b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:50,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742002_1178 (size=12663) 2024-12-03T04:43:50,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742003_1179 (size=12454) 2024-12-03T04:43:50,617 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/62da020adf344a50ab911ce8e7e3af05 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/62da020adf344a50ab911ce8e7e3af05 2024-12-03T04:43:50,623 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/B of ad1a6acd12c06a97ea4048034fea205b into 62da020adf344a50ab911ce8e7e3af05(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:50,623 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:50,623 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/B, priority=11, startTime=1733201030479; duration=0sec 2024-12-03T04:43:50,623 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:50,623 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:B 2024-12-03T04:43:50,623 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T04:43:50,626 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61097 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T04:43:50,627 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/C is initiating minor compaction (all files) 2024-12-03T04:43:50,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742004_1180 (size=4469) 2024-12-03T04:43:50,627 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/C in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:50,627 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/e16bb56e376947bd90f3007d40d32358, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ae93a879701a44ebab495b71fdd118fd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/891b828b84ea4e1e8cdd687a868e7f42, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/c52dc0c9112b481a9a1125afcfeee47b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/af7553a201684facac4e8115211c3649] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=59.7 K 2024-12-03T04:43:50,629 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e16bb56e376947bd90f3007d40d32358, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733201024483 2024-12-03T04:43:50,629 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#A#compaction#148 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:50,629 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ae93a879701a44ebab495b71fdd118fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733201025510 2024-12-03T04:43:50,630 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/1dc9775f29114a6591cbc524535009be is 175, key is test_row_0/A:col10/1733201028735/Put/seqid=0 2024-12-03T04:43:50,630 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 891b828b84ea4e1e8cdd687a868e7f42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733201026137 2024-12-03T04:43:50,631 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c52dc0c9112b481a9a1125afcfeee47b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733201027446 2024-12-03T04:43:50,631 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting af7553a201684facac4e8115211c3649, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733201028735 2024-12-03T04:43:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T04:43:50,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742005_1181 (size=31617) 2024-12-03T04:43:50,692 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/1dc9775f29114a6591cbc524535009be as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/1dc9775f29114a6591cbc524535009be 2024-12-03T04:43:50,699 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#C#compaction#150 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:50,700 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/781937d2da1245758c025b700c23b83f is 50, key is test_row_0/C:col10/1733201028735/Put/seqid=0 2024-12-03T04:43:50,701 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/A of ad1a6acd12c06a97ea4048034fea205b into 1dc9775f29114a6591cbc524535009be(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:50,701 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:50,701 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/A, priority=11, startTime=1733201030479; duration=0sec 2024-12-03T04:43:50,701 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:50,701 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:A 2024-12-03T04:43:50,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742006_1182 (size=12663) 2024-12-03T04:43:50,751 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/781937d2da1245758c025b700c23b83f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/781937d2da1245758c025b700c23b83f 2024-12-03T04:43:50,760 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/C of ad1a6acd12c06a97ea4048034fea205b into 781937d2da1245758c025b700c23b83f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:50,760 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:50,760 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/C, priority=11, startTime=1733201030479; duration=0sec 2024-12-03T04:43:50,760 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:50,760 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:C 2024-12-03T04:43:50,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:50,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201090809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:50,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201090812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:50,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201090922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201090922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:51,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:51,022 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f3eb36c298af4ae59e48788e89f473c3_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f3eb36c298af4ae59e48788e89f473c3_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/552889529f724beba7d2a8048ba328a0, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:51,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/552889529f724beba7d2a8048ba328a0 is 175, key is test_row_0/A:col10/1733201029614/Put/seqid=0 2024-12-03T04:43:51,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742007_1183 (size=31255) 2024-12-03T04:43:51,048 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/552889529f724beba7d2a8048ba328a0 2024-12-03T04:43:51,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/88c9f3cb0951422f922497a210f2c700 is 50, key is test_row_0/B:col10/1733201029614/Put/seqid=0 2024-12-03T04:43:51,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742008_1184 (size=12301) 2024-12-03T04:43:51,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201091133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:51,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:51,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:51,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:51,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201091133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:51,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:51,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:51,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201091452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:51,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201091452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:51,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:51,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:51,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:51,525 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/88c9f3cb0951422f922497a210f2c700 2024-12-03T04:43:51,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/3303a26b245f48f3bd79f6c2a0ccba13 is 50, key is test_row_0/C:col10/1733201029614/Put/seqid=0 2024-12-03T04:43:51,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742009_1185 (size=12301) 2024-12-03T04:43:51,585 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/3303a26b245f48f3bd79f6c2a0ccba13 2024-12-03T04:43:51,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/552889529f724beba7d2a8048ba328a0 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/552889529f724beba7d2a8048ba328a0 2024-12-03T04:43:51,620 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/552889529f724beba7d2a8048ba328a0, entries=150, sequenceid=267, filesize=30.5 K 2024-12-03T04:43:51,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/88c9f3cb0951422f922497a210f2c700 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/88c9f3cb0951422f922497a210f2c700 2024-12-03T04:43:51,629 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/88c9f3cb0951422f922497a210f2c700, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T04:43:51,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/3303a26b245f48f3bd79f6c2a0ccba13 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/3303a26b245f48f3bd79f6c2a0ccba13 2024-12-03T04:43:51,657 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/3303a26b245f48f3bd79f6c2a0ccba13, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T04:43:51,662 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for ad1a6acd12c06a97ea4048034fea205b in 1177ms, sequenceid=267, compaction requested=false 2024-12-03T04:43:51,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:51,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:51,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-03T04:43:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-03T04:43:51,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-03T04:43:51,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1210 sec 2024-12-03T04:43:51,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.1310 sec 2024-12-03T04:43:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T04:43:51,706 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:51,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-03T04:43:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T04:43:51,709 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:51,710 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:51,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T04:43:51,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T04:43:51,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:51,865 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T04:43:51,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:51,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:51,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:51,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:51,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:51,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:51,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d760c211c53c41108add100732645084_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201030801/Put/seqid=0 2024-12-03T04:43:51,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742010_1186 (size=12454) 2024-12-03T04:43:51,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:52,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T04:43:52,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201032040, max timestamp=1733201032095Num uniques 859; 2024-12-03T04:43:52,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201032040, max timestamp=1733201032095Num uniques 859; 2024-12-03T04:43:52,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:52,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:52,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201032040, max timestamp=1733201032095Num uniques 859; 2024-12-03T04:43:52,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:43:52,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201092158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:52,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:52,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201092160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201092272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201092272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:52,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:52,315 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d760c211c53c41108add100732645084_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d760c211c53c41108add100732645084_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:52,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/5e6ac0f5a0044cc9a9ebcd1bcc897486, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:52,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/5e6ac0f5a0044cc9a9ebcd1bcc897486 is 175, key is test_row_0/A:col10/1733201030801/Put/seqid=0 2024-12-03T04:43:52,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742011_1187 (size=31255) 2024-12-03T04:43:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T04:43:52,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201092482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:52,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201092483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:52,727 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/5e6ac0f5a0044cc9a9ebcd1bcc897486 2024-12-03T04:43:52,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/5d04206f7c804ea08966961130b285ff is 50, key is test_row_0/B:col10/1733201030801/Put/seqid=0 2024-12-03T04:43:52,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742012_1188 (size=12301) 2024-12-03T04:43:52,758 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/5d04206f7c804ea08966961130b285ff 2024-12-03T04:43:52,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58104 deadline: 1733201092771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:52,775 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in ad1a6acd12c06a97ea4048034fea205b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:52,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:52,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/ebd73f14a6664c3c98bec45ebd12338e is 50, key is test_row_0/C:col10/1733201030801/Put/seqid=0 2024-12-03T04:43:52,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742013_1189 (size=12301) 2024-12-03T04:43:52,788 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/ebd73f14a6664c3c98bec45ebd12338e 2024-12-03T04:43:52,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/5e6ac0f5a0044cc9a9ebcd1bcc897486 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/5e6ac0f5a0044cc9a9ebcd1bcc897486 2024-12-03T04:43:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201092802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58086 deadline: 1733201092802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:52,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201092802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:52,807 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in ad1a6acd12c06a97ea4048034fea205b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:52,809 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/5e6ac0f5a0044cc9a9ebcd1bcc897486, entries=150, sequenceid=281, filesize=30.5 K 2024-12-03T04:43:52,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/5d04206f7c804ea08966961130b285ff as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/5d04206f7c804ea08966961130b285ff 2024-12-03T04:43:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58096 deadline: 1733201092812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:52,818 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in ad1a6acd12c06a97ea4048034fea205b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8317 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:43:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:52,822 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/5d04206f7c804ea08966961130b285ff, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T04:43:52,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/ebd73f14a6664c3c98bec45ebd12338e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ebd73f14a6664c3c98bec45ebd12338e 2024-12-03T04:43:52,832 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ebd73f14a6664c3c98bec45ebd12338e, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T04:43:52,833 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ad1a6acd12c06a97ea4048034fea205b in 969ms, sequenceid=281, compaction requested=true 2024-12-03T04:43:52,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:52,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:52,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-03T04:43:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-03T04:43:52,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-03T04:43:52,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1270 sec 2024-12-03T04:43:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T04:43:52,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.1350 sec 2024-12-03T04:43:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:53,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-03T04:43:53,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:53,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:53,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:53,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:53,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:53,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:53,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120333d0f6a5655b4e52b2e52b3fc1defdfe_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201033313/Put/seqid=0 2024-12-03T04:43:53,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742014_1190 (size=14994) 2024-12-03T04:43:53,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201093359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:53,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201093363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:53,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:53,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:53,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201093484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:53,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201093486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:53,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201093702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:53,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201093705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:53,764 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:53,773 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120333d0f6a5655b4e52b2e52b3fc1defdfe_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120333d0f6a5655b4e52b2e52b3fc1defdfe_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:53,775 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/db11d9d6f38c43c9b26616926fb9ea96, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:53,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/db11d9d6f38c43c9b26616926fb9ea96 is 175, key is test_row_0/A:col10/1733201033313/Put/seqid=0 2024-12-03T04:43:53,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742015_1191 (size=39949) 2024-12-03T04:43:53,799 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/db11d9d6f38c43c9b26616926fb9ea96 2024-12-03T04:43:53,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/6460be02655f4963bc74ed1e0e341b5b is 50, key is test_row_0/B:col10/1733201033313/Put/seqid=0 2024-12-03T04:43:53,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742016_1192 (size=12301) 2024-12-03T04:43:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T04:43:53,852 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:53,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:43:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-03T04:43:53,854 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:43:53,856 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:43:53,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:43:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T04:43:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T04:43:54,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T04:43:54,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:54,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201094011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:54,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201094022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:54,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T04:43:54,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T04:43:54,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/6460be02655f4963bc74ed1e0e341b5b 2024-12-03T04:43:54,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/dff0195cd0374561ade01a6ed2c777f5 is 50, key is test_row_0/C:col10/1733201033313/Put/seqid=0 2024-12-03T04:43:54,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742017_1193 (size=12301) 2024-12-03T04:43:54,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T04:43:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T04:43:54,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:54,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T04:43:54,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201094522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:54,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:54,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:54,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:54,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201094532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:54,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T04:43:54,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:54,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:43:54,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/dff0195cd0374561ade01a6ed2c777f5 2024-12-03T04:43:54,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/db11d9d6f38c43c9b26616926fb9ea96 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/db11d9d6f38c43c9b26616926fb9ea96 2024-12-03T04:43:54,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/db11d9d6f38c43c9b26616926fb9ea96, entries=200, sequenceid=317, filesize=39.0 K 2024-12-03T04:43:54,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/6460be02655f4963bc74ed1e0e341b5b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6460be02655f4963bc74ed1e0e341b5b 2024-12-03T04:43:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6460be02655f4963bc74ed1e0e341b5b, entries=150, sequenceid=317, filesize=12.0 K 2024-12-03T04:43:54,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/dff0195cd0374561ade01a6ed2c777f5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/dff0195cd0374561ade01a6ed2c777f5 2024-12-03T04:43:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/dff0195cd0374561ade01a6ed2c777f5, entries=150, sequenceid=317, filesize=12.0 K 2024-12-03T04:43:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for ad1a6acd12c06a97ea4048034fea205b in 1366ms, sequenceid=317, compaction requested=true 2024-12-03T04:43:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:54,683 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:54,684 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:54,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:54,685 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134076 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:54,685 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/A is initiating minor compaction (all files) 2024-12-03T04:43:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,685 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/A in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,685 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/1dc9775f29114a6591cbc524535009be, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/552889529f724beba7d2a8048ba328a0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/5e6ac0f5a0044cc9a9ebcd1bcc897486, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/db11d9d6f38c43c9b26616926fb9ea96] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=130.9 K 2024-12-03T04:43:54,685 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,685 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/1dc9775f29114a6591cbc524535009be, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/552889529f724beba7d2a8048ba328a0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/5e6ac0f5a0044cc9a9ebcd1bcc897486, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/db11d9d6f38c43c9b26616926fb9ea96] 2024-12-03T04:43:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,687 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dc9775f29114a6591cbc524535009be, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733201028735 2024-12-03T04:43:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,688 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:54,688 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 552889529f724beba7d2a8048ba328a0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733201029614 2024-12-03T04:43:54,688 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/B is initiating minor compaction (all files) 2024-12-03T04:43:54,688 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/B in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,689 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/62da020adf344a50ab911ce8e7e3af05, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/88c9f3cb0951422f922497a210f2c700, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/5d04206f7c804ea08966961130b285ff, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6460be02655f4963bc74ed1e0e341b5b] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=48.4 K 2024-12-03T04:43:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,690 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e6ac0f5a0044cc9a9ebcd1bcc897486, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733201030801 2024-12-03T04:43:54,690 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 62da020adf344a50ab911ce8e7e3af05, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733201028735 2024-12-03T04:43:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,691 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting db11d9d6f38c43c9b26616926fb9ea96, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201032095 2024-12-03T04:43:54,691 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 88c9f3cb0951422f922497a210f2c700, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733201029614 2024-12-03T04:43:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,692 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d04206f7c804ea08966961130b285ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733201030801 2024-12-03T04:43:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,692 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 6460be02655f4963bc74ed1e0e341b5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201032152 2024-12-03T04:43:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,703 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,710 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203d0e41adf8d494479afa1c8faceb69636_ad1a6acd12c06a97ea4048034fea205b store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:54,712 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:54,713 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203d0e41adf8d494479afa1c8faceb69636_ad1a6acd12c06a97ea4048034fea205b, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,713 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d0e41adf8d494479afa1c8faceb69636_ad1a6acd12c06a97ea4048034fea205b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:54,713 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/a8252349e1bf4473ae67d6d99f9a848f is 50, key is test_row_0/B:col10/1733201033313/Put/seqid=0 2024-12-03T04:43:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742019_1195 (size=12949) 2024-12-03T04:43:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742018_1194 (size=4469) 2024-12-03T04:43:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,732 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#A#compaction#159 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,733 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/d512c5f3b3ed4a15a013adafa75744b6 is 175, key is test_row_0/A:col10/1733201033313/Put/seqid=0 2024-12-03T04:43:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742020_1196 (size=31903) 2024-12-03T04:43:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T04:43:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:54,790 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T04:43:54,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:54,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:54,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:54,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:54,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036206a33c4b354113827c91d9e7a6f769_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201033354/Put/seqid=0 2024-12-03T04:43:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742021_1197 (size=9914) 2024-12-03T04:43:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T04:43:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,138 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/a8252349e1bf4473ae67d6d99f9a848f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/a8252349e1bf4473ae67d6d99f9a848f 2024-12-03T04:43:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,147 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/B of ad1a6acd12c06a97ea4048034fea205b into a8252349e1bf4473ae67d6d99f9a848f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:55,147 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:55,147 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/B, priority=12, startTime=1733201034684; duration=0sec 2024-12-03T04:43:55,147 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:55,147 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:B 2024-12-03T04:43:55,147 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:43:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,149 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:43:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,149 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/C is initiating minor compaction (all files) 2024-12-03T04:43:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,149 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/C in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:55,149 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/781937d2da1245758c025b700c23b83f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/3303a26b245f48f3bd79f6c2a0ccba13, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ebd73f14a6664c3c98bec45ebd12338e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/dff0195cd0374561ade01a6ed2c777f5] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=48.4 K 2024-12-03T04:43:55,150 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 781937d2da1245758c025b700c23b83f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733201028735 2024-12-03T04:43:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,150 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3303a26b245f48f3bd79f6c2a0ccba13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733201029614 2024-12-03T04:43:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,151 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ebd73f14a6664c3c98bec45ebd12338e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733201030801 2024-12-03T04:43:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,152 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting dff0195cd0374561ade01a6ed2c777f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201032152 2024-12-03T04:43:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,164 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/d512c5f3b3ed4a15a013adafa75744b6 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d512c5f3b3ed4a15a013adafa75744b6 2024-12-03T04:43:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,170 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/A of ad1a6acd12c06a97ea4048034fea205b into d512c5f3b3ed4a15a013adafa75744b6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:55,170 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:55,170 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/A, priority=12, startTime=1733201034683; duration=0sec 2024-12-03T04:43:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,170 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:55,171 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:A 2024-12-03T04:43:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,176 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,177 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/5f0a9814a9ca49a0b8f4900c98338eb4 is 50, key is test_row_0/C:col10/1733201033313/Put/seqid=0 2024-12-03T04:43:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742022_1198 (size=12949) 2024-12-03T04:43:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,241 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036206a33c4b354113827c91d9e7a6f769_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036206a33c4b354113827c91d9e7a6f769_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/0f47e88e8afd4d28987038cdfdf0e663, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/0f47e88e8afd4d28987038cdfdf0e663 is 175, key is test_row_0/A:col10/1733201033354/Put/seqid=0 2024-12-03T04:43:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742023_1199 (size=22561) 2024-12-03T04:43:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. as already flushing 2024-12-03T04:43:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:55,662 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/5f0a9814a9ca49a0b8f4900c98338eb4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/5f0a9814a9ca49a0b8f4900c98338eb4 2024-12-03T04:43:55,670 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/C of ad1a6acd12c06a97ea4048034fea205b into 5f0a9814a9ca49a0b8f4900c98338eb4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:55,670 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:55,670 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/C, priority=12, startTime=1733201034684; duration=0sec 2024-12-03T04:43:55,671 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:55,671 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:C 2024-12-03T04:43:55,678 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/0f47e88e8afd4d28987038cdfdf0e663 2024-12-03T04:43:55,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201035635, max timestamp=1733201035681Num uniques 859; 2024-12-03T04:43:55,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:55,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201035635, max timestamp=1733201035681Num uniques 859; 2024-12-03T04:43:55,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:55,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201035635, max timestamp=1733201035681Num uniques 859; 2024-12-03T04:43:55,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:43:55,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/a55496585eb34777bacc976157eb771a is 50, key is test_row_0/B:col10/1733201033354/Put/seqid=0 2024-12-03T04:43:55,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742024_1200 (size=9857) 2024-12-03T04:43:55,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201095764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:55,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201095769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:55,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201095882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:55,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201095882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T04:43:56,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201096102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:56,112 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/a55496585eb34777bacc976157eb771a 2024-12-03T04:43:56,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201096102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:56,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/f1dbfd4a8f114d579003f0a006bd97b7 is 50, key is test_row_0/C:col10/1733201033354/Put/seqid=0 2024-12-03T04:43:56,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742025_1201 (size=9857) 2024-12-03T04:43:56,162 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/f1dbfd4a8f114d579003f0a006bd97b7 2024-12-03T04:43:56,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/0f47e88e8afd4d28987038cdfdf0e663 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/0f47e88e8afd4d28987038cdfdf0e663 2024-12-03T04:43:56,182 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/0f47e88e8afd4d28987038cdfdf0e663, entries=100, sequenceid=328, filesize=22.0 K 2024-12-03T04:43:56,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/a55496585eb34777bacc976157eb771a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/a55496585eb34777bacc976157eb771a 2024-12-03T04:43:56,200 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/a55496585eb34777bacc976157eb771a, entries=100, sequenceid=328, filesize=9.6 K 2024-12-03T04:43:56,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/f1dbfd4a8f114d579003f0a006bd97b7 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f1dbfd4a8f114d579003f0a006bd97b7 2024-12-03T04:43:56,208 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f1dbfd4a8f114d579003f0a006bd97b7, entries=100, sequenceid=328, filesize=9.6 K 2024-12-03T04:43:56,212 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ad1a6acd12c06a97ea4048034fea205b in 1421ms, sequenceid=328, compaction requested=false 2024-12-03T04:43:56,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:56,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:56,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-03T04:43:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-03T04:43:56,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-03T04:43:56,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3580 sec 2024-12-03T04:43:56,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.3650 sec 2024-12-03T04:43:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:56,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-03T04:43:56,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:56,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733201035726, max timestamp=1733201036425Num uniques 1018; 2024-12-03T04:43:56,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:43:56,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-03T04:43:56,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:56,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:43:56,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:56,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:43:56,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032b22b9c9006a4f09a8e880ab9679a2b2_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201035726/Put/seqid=0 2024-12-03T04:43:56,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201096535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:56,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201096533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:56,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742026_1202 (size=17534) 2024-12-03T04:43:56,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201096652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201096652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:56,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201096862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:56,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:56,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201096863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:56,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:56,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:56,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:56,959 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:56,965 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032b22b9c9006a4f09a8e880ab9679a2b2_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032b22b9c9006a4f09a8e880ab9679a2b2_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:56,966 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/10f743bd51c24ce482de30e06552fca1, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:56,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/10f743bd51c24ce482de30e06552fca1 is 175, key is test_row_0/A:col10/1733201035726/Put/seqid=0 2024-12-03T04:43:56,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742027_1203 (size=48639) 2024-12-03T04:43:57,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201097172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:57,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201097182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:57,382 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/10f743bd51c24ce482de30e06552fca1 2024-12-03T04:43:57,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/3680d51c0e0148dba4981c97eaa8e19b is 50, key is test_row_0/B:col10/1733201035726/Put/seqid=0 2024-12-03T04:43:57,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742028_1204 (size=12301) 2024-12-03T04:43:57,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/3680d51c0e0148dba4981c97eaa8e19b 2024-12-03T04:43:57,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/4ba03aae61c044f6b631532961c79faa is 50, key is test_row_0/C:col10/1733201035726/Put/seqid=0 2024-12-03T04:43:57,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742029_1205 (size=12301) 2024-12-03T04:43:57,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58120 deadline: 1733201097681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:57,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:57,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:57,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:57,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:43:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58122 deadline: 1733201097691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 2024-12-03T04:43:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:43:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:43:57,756 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-12-03T04:43:57,756 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:57,756 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:57,756 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:57,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:57,761 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-12-03T04:43:57,761 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:57,761 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:57,761 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:57,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:57,767 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-12-03T04:43:57,767 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:57,767 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:57,767 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:57,767 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-12-03T04:43:57,767 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:57,767 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:57,767 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:57,767 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:57,767 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:57,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/4ba03aae61c044f6b631532961c79faa 2024-12-03T04:43:57,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/10f743bd51c24ce482de30e06552fca1 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/10f743bd51c24ce482de30e06552fca1 2024-12-03T04:43:57,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/10f743bd51c24ce482de30e06552fca1, entries=250, sequenceid=369, filesize=47.5 K 2024-12-03T04:43:57,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/3680d51c0e0148dba4981c97eaa8e19b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/3680d51c0e0148dba4981c97eaa8e19b 2024-12-03T04:43:57,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/3680d51c0e0148dba4981c97eaa8e19b, entries=150, sequenceid=369, filesize=12.0 K 2024-12-03T04:43:57,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/4ba03aae61c044f6b631532961c79faa as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/4ba03aae61c044f6b631532961c79faa 2024-12-03T04:43:57,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/4ba03aae61c044f6b631532961c79faa, entries=150, sequenceid=369, filesize=12.0 K 2024-12-03T04:43:57,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~105.11 KB/107630, heapSize ~297.23 KB/304368, currentSize=73.80 KB/75570 for ad1a6acd12c06a97ea4048034fea205b in 1440ms, sequenceid=369, compaction requested=true 2024-12-03T04:43:57,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:57,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:43:57,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:57,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:43:57,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:57,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ad1a6acd12c06a97ea4048034fea205b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:43:57,878 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:57,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:57,878 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:57,879 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:57,879 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103103 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:57,879 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/B is initiating minor compaction (all files) 2024-12-03T04:43:57,879 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/A is initiating minor compaction (all files) 2024-12-03T04:43:57,879 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/B in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:57,879 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/A in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:57,879 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/a8252349e1bf4473ae67d6d99f9a848f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/a55496585eb34777bacc976157eb771a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/3680d51c0e0148dba4981c97eaa8e19b] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=34.3 K 2024-12-03T04:43:57,879 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d512c5f3b3ed4a15a013adafa75744b6, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/0f47e88e8afd4d28987038cdfdf0e663, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/10f743bd51c24ce482de30e06552fca1] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=100.7 K 2024-12-03T04:43:57,879 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:57,880 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d512c5f3b3ed4a15a013adafa75744b6, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/0f47e88e8afd4d28987038cdfdf0e663, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/10f743bd51c24ce482de30e06552fca1] 2024-12-03T04:43:57,880 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a8252349e1bf4473ae67d6d99f9a848f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201032152 2024-12-03T04:43:57,880 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting d512c5f3b3ed4a15a013adafa75744b6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201032152 2024-12-03T04:43:57,880 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a55496585eb34777bacc976157eb771a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733201033354 2024-12-03T04:43:57,880 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f47e88e8afd4d28987038cdfdf0e663, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733201033354 2024-12-03T04:43:57,881 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3680d51c0e0148dba4981c97eaa8e19b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733201035726 2024-12-03T04:43:57,881 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10f743bd51c24ce482de30e06552fca1, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733201035726 2024-12-03T04:43:57,890 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#B#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:57,891 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/1370404ac36d4e80971a489c977784b1 is 50, key is test_row_0/B:col10/1733201035726/Put/seqid=0 2024-12-03T04:43:57,893 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:57,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742030_1206 (size=13051) 2024-12-03T04:43:57,895 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203d3dfd0d10ba54ef6940e469398d52b8d_ad1a6acd12c06a97ea4048034fea205b store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:57,921 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203d3dfd0d10ba54ef6940e469398d52b8d_ad1a6acd12c06a97ea4048034fea205b, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:57,921 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d3dfd0d10ba54ef6940e469398d52b8d_ad1a6acd12c06a97ea4048034fea205b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:57,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742031_1207 (size=4469) 2024-12-03T04:43:57,936 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#A#compaction#169 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:57,937 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/2a43947329524d0ea441355ff3e3d11c is 175, key is test_row_0/A:col10/1733201035726/Put/seqid=0 2024-12-03T04:43:57,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742032_1208 (size=32005) 2024-12-03T04:43:57,947 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/2a43947329524d0ea441355ff3e3d11c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2a43947329524d0ea441355ff3e3d11c 2024-12-03T04:43:57,953 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/A of ad1a6acd12c06a97ea4048034fea205b into 2a43947329524d0ea441355ff3e3d11c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:57,953 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:57,953 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/A, priority=13, startTime=1733201037878; duration=0sec 2024-12-03T04:43:57,953 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:43:57,953 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:A 2024-12-03T04:43:57,953 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:43:57,954 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:43:57,954 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): ad1a6acd12c06a97ea4048034fea205b/C is initiating minor compaction (all files) 2024-12-03T04:43:57,954 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ad1a6acd12c06a97ea4048034fea205b/C in TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:43:57,954 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/5f0a9814a9ca49a0b8f4900c98338eb4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f1dbfd4a8f114d579003f0a006bd97b7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/4ba03aae61c044f6b631532961c79faa] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp, totalSize=34.3 K 2024-12-03T04:43:57,955 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f0a9814a9ca49a0b8f4900c98338eb4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201032152 2024-12-03T04:43:57,955 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1dbfd4a8f114d579003f0a006bd97b7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733201033354 2024-12-03T04:43:57,955 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ba03aae61c044f6b631532961c79faa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733201035726 2024-12-03T04:43:57,966 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ad1a6acd12c06a97ea4048034fea205b#C#compaction#170 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:43:57,967 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/50fdad212d264a6491d5b43ebe97922e is 50, key is test_row_0/C:col10/1733201035726/Put/seqid=0 2024-12-03T04:43:57,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742033_1209 (size=13051) 2024-12-03T04:43:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T04:43:58,022 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:43:58,300 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/1370404ac36d4e80971a489c977784b1 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/1370404ac36d4e80971a489c977784b1 2024-12-03T04:43:58,304 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/B of ad1a6acd12c06a97ea4048034fea205b into 1370404ac36d4e80971a489c977784b1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:58,304 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:58,305 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/B, priority=13, startTime=1733201037878; duration=0sec 2024-12-03T04:43:58,305 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:58,305 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:B 2024-12-03T04:43:58,379 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/50fdad212d264a6491d5b43ebe97922e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/50fdad212d264a6491d5b43ebe97922e 2024-12-03T04:43:58,385 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ad1a6acd12c06a97ea4048034fea205b/C of ad1a6acd12c06a97ea4048034fea205b into 50fdad212d264a6491d5b43ebe97922e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:43:58,385 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:43:58,385 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., storeName=ad1a6acd12c06a97ea4048034fea205b/C, priority=13, startTime=1733201037878; duration=0sec 2024-12-03T04:43:58,385 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:43:58,385 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ad1a6acd12c06a97ea4048034fea205b:C 2024-12-03T04:43:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:58,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T04:43:58,704 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-12-03T04:43:58,704 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-12-03T04:43:58,704 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:58,704 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:43:58,704 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:58,704 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:58,704 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:58,704 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:43:58,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:43:58,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:58,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:58,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:43:58,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:58,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:43:58,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:43:58,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:43:58,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035b971e9ee0c2457e8bb667bcbeb57914_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_0/A:col10/1733201038702/Put/seqid=0 2024-12-03T04:43:58,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742034_1210 (size=12454) 2024-12-03T04:43:59,124 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:43:59,129 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035b971e9ee0c2457e8bb667bcbeb57914_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035b971e9ee0c2457e8bb667bcbeb57914_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:43:59,130 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/2817e6fccb1240988dc19331c2291ae4, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:43:59,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/2817e6fccb1240988dc19331c2291ae4 is 175, key is test_row_0/A:col10/1733201038702/Put/seqid=0 2024-12-03T04:43:59,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742035_1211 (size=31255) 2024-12-03T04:43:59,538 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/2817e6fccb1240988dc19331c2291ae4 2024-12-03T04:43:59,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b4575ab32d7741398b25c54b2f2c7968 is 50, key is test_row_0/B:col10/1733201038702/Put/seqid=0 2024-12-03T04:43:59,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742036_1212 (size=12301) 2024-12-03T04:43:59,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b4575ab32d7741398b25c54b2f2c7968 2024-12-03T04:43:59,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/d3df8776c2e94270955e08fe50bce197 is 50, key is test_row_0/C:col10/1733201038702/Put/seqid=0 2024-12-03T04:43:59,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742037_1213 (size=12301) 2024-12-03T04:44:00,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/d3df8776c2e94270955e08fe50bce197 2024-12-03T04:44:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/2817e6fccb1240988dc19331c2291ae4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2817e6fccb1240988dc19331c2291ae4 2024-12-03T04:44:00,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2817e6fccb1240988dc19331c2291ae4, entries=150, sequenceid=388, filesize=30.5 K 2024-12-03T04:44:00,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/b4575ab32d7741398b25c54b2f2c7968 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b4575ab32d7741398b25c54b2f2c7968 2024-12-03T04:44:00,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b4575ab32d7741398b25c54b2f2c7968, entries=150, sequenceid=388, filesize=12.0 K 2024-12-03T04:44:00,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/d3df8776c2e94270955e08fe50bce197 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/d3df8776c2e94270955e08fe50bce197 2024-12-03T04:44:00,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/d3df8776c2e94270955e08fe50bce197, entries=150, sequenceid=388, filesize=12.0 K 2024-12-03T04:44:00,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for ad1a6acd12c06a97ea4048034fea205b in 1687ms, sequenceid=388, compaction requested=false 2024-12-03T04:44:00,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ad1a6acd12c06a97ea4048034fea205b: 2024-12-03T04:44:02,823 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-12-03T04:44:02,823 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:02,823 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:02,824 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:02,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:02,833 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-12-03T04:44:02,833 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:02,833 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:02,833 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:02,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:02,873 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-12-03T04:44:02,873 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:02,873 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:02,874 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:02,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1533 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1508 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 721 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2152 rows 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 720 2024-12-03T04:44:02,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2144 rows 2024-12-03T04:44:02,874 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T04:44:02,874 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T04:44:02,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:02,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:02,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:02,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T04:44:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T04:44:02,879 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201042879"}]},"ts":"1733201042879"} 2024-12-03T04:44:02,881 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T04:44:02,881 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T04:44:02,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T04:44:02,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, UNASSIGN}] 2024-12-03T04:44:02,884 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, UNASSIGN 2024-12-03T04:44:02,885 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=ad1a6acd12c06a97ea4048034fea205b, regionState=CLOSING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:44:02,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, UNASSIGN because future has completed 2024-12-03T04:44:02,887 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T04:44:02,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:44:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T04:44:03,040 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:03,040 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T04:44:03,040 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing ad1a6acd12c06a97ea4048034fea205b, disabling compactions & flushes 2024-12-03T04:44:03,040 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:44:03,040 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:44:03,040 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. after waiting 0 ms 2024-12-03T04:44:03,040 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:44:03,041 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing ad1a6acd12c06a97ea4048034fea205b 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-03T04:44:03,041 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=A 2024-12-03T04:44:03,041 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:03,041 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=B 2024-12-03T04:44:03,041 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:03,041 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ad1a6acd12c06a97ea4048034fea205b, store=C 2024-12-03T04:44:03,041 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:03,075 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203881e408f709544779ee41850fd574e5b_ad1a6acd12c06a97ea4048034fea205b is 50, key is test_row_1/A:col10/1733201042872/Put/seqid=0 2024-12-03T04:44:03,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742038_1214 (size=9914) 2024-12-03T04:44:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T04:44:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T04:44:03,506 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:03,512 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203881e408f709544779ee41850fd574e5b_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203881e408f709544779ee41850fd574e5b_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:03,514 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/b3016785667443cca469a1be8328e0d5, store: [table=TestAcidGuarantees family=A region=ad1a6acd12c06a97ea4048034fea205b] 2024-12-03T04:44:03,515 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/b3016785667443cca469a1be8328e0d5 is 175, key is test_row_1/A:col10/1733201042872/Put/seqid=0 2024-12-03T04:44:03,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742039_1215 (size=22561) 2024-12-03T04:44:03,932 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/b3016785667443cca469a1be8328e0d5 2024-12-03T04:44:03,941 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/4f8b52aa43aa4b0db3ae09f7b733e903 is 50, key is test_row_1/B:col10/1733201042872/Put/seqid=0 2024-12-03T04:44:03,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742040_1216 (size=9857) 2024-12-03T04:44:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T04:44:04,355 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/4f8b52aa43aa4b0db3ae09f7b733e903 2024-12-03T04:44:04,365 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/7bdcafa8d223475ca71b5d86762f4726 is 50, key is test_row_1/C:col10/1733201042872/Put/seqid=0 2024-12-03T04:44:04,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742041_1217 (size=9857) 2024-12-03T04:44:04,781 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/7bdcafa8d223475ca71b5d86762f4726 2024-12-03T04:44:04,788 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/A/b3016785667443cca469a1be8328e0d5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/b3016785667443cca469a1be8328e0d5 2024-12-03T04:44:04,795 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/b3016785667443cca469a1be8328e0d5, entries=100, sequenceid=394, filesize=22.0 K 2024-12-03T04:44:04,798 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/B/4f8b52aa43aa4b0db3ae09f7b733e903 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/4f8b52aa43aa4b0db3ae09f7b733e903 2024-12-03T04:44:04,804 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/4f8b52aa43aa4b0db3ae09f7b733e903, entries=100, sequenceid=394, filesize=9.6 K 2024-12-03T04:44:04,805 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/.tmp/C/7bdcafa8d223475ca71b5d86762f4726 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/7bdcafa8d223475ca71b5d86762f4726 2024-12-03T04:44:04,809 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/7bdcafa8d223475ca71b5d86762f4726, entries=100, sequenceid=394, filesize=9.6 K 2024-12-03T04:44:04,810 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ad1a6acd12c06a97ea4048034fea205b in 1770ms, sequenceid=394, compaction requested=true 2024-12-03T04:44:04,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/262ec4bbf83b435aaf5ab2218740a7f5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/76c122cbfc844fafb1a9d806a944ac9c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/97dab83d37e74f9fa66d2fd344607ab3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/45def0c6fb504877a1504f2c8e306bef, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/ddea4ad9bf52480b89524ef4a5b75e1f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2d44e3e32b80433088e96bd45485fd42, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85a3d0063cfb468c8ab94252c6cef53e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/142c6a6469074250abcaa220033b0df9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/aefacd6629da41c1bece474710853c4f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d50f2c485add4296af880f3eda077200, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/68bc2f877434440e90cd72a76dd9dd64, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/6ba7a23f55de4618b866ea907292ae92, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/e5b0654e3aa24f8c9588e0969d86e0ae, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85b48c3d9a7f40eba1ba945871d1adfe, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/1dc9775f29114a6591cbc524535009be, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/18993899492d480da5578ed72052f3fd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/552889529f724beba7d2a8048ba328a0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/5e6ac0f5a0044cc9a9ebcd1bcc897486, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/db11d9d6f38c43c9b26616926fb9ea96, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d512c5f3b3ed4a15a013adafa75744b6, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/0f47e88e8afd4d28987038cdfdf0e663, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/10f743bd51c24ce482de30e06552fca1] to archive 2024-12-03T04:44:04,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:44:04,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/262ec4bbf83b435aaf5ab2218740a7f5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/262ec4bbf83b435aaf5ab2218740a7f5 2024-12-03T04:44:04,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/76c122cbfc844fafb1a9d806a944ac9c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/76c122cbfc844fafb1a9d806a944ac9c 2024-12-03T04:44:04,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/97dab83d37e74f9fa66d2fd344607ab3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/97dab83d37e74f9fa66d2fd344607ab3 2024-12-03T04:44:04,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/45def0c6fb504877a1504f2c8e306bef to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/45def0c6fb504877a1504f2c8e306bef 2024-12-03T04:44:04,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/ddea4ad9bf52480b89524ef4a5b75e1f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/ddea4ad9bf52480b89524ef4a5b75e1f 2024-12-03T04:44:04,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2d44e3e32b80433088e96bd45485fd42 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2d44e3e32b80433088e96bd45485fd42 2024-12-03T04:44:04,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85a3d0063cfb468c8ab94252c6cef53e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85a3d0063cfb468c8ab94252c6cef53e 2024-12-03T04:44:04,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/142c6a6469074250abcaa220033b0df9 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/142c6a6469074250abcaa220033b0df9 2024-12-03T04:44:04,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/aefacd6629da41c1bece474710853c4f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/aefacd6629da41c1bece474710853c4f 2024-12-03T04:44:04,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d50f2c485add4296af880f3eda077200 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d50f2c485add4296af880f3eda077200 2024-12-03T04:44:04,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/68bc2f877434440e90cd72a76dd9dd64 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/68bc2f877434440e90cd72a76dd9dd64 2024-12-03T04:44:04,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/6ba7a23f55de4618b866ea907292ae92 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/6ba7a23f55de4618b866ea907292ae92 2024-12-03T04:44:04,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/e5b0654e3aa24f8c9588e0969d86e0ae to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/e5b0654e3aa24f8c9588e0969d86e0ae 2024-12-03T04:44:04,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85b48c3d9a7f40eba1ba945871d1adfe to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/85b48c3d9a7f40eba1ba945871d1adfe 2024-12-03T04:44:04,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/1dc9775f29114a6591cbc524535009be to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/1dc9775f29114a6591cbc524535009be 2024-12-03T04:44:04,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/18993899492d480da5578ed72052f3fd to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/18993899492d480da5578ed72052f3fd 2024-12-03T04:44:04,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/552889529f724beba7d2a8048ba328a0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/552889529f724beba7d2a8048ba328a0 2024-12-03T04:44:04,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/5e6ac0f5a0044cc9a9ebcd1bcc897486 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/5e6ac0f5a0044cc9a9ebcd1bcc897486 2024-12-03T04:44:04,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/db11d9d6f38c43c9b26616926fb9ea96 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/db11d9d6f38c43c9b26616926fb9ea96 2024-12-03T04:44:04,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d512c5f3b3ed4a15a013adafa75744b6 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/d512c5f3b3ed4a15a013adafa75744b6 2024-12-03T04:44:04,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/0f47e88e8afd4d28987038cdfdf0e663 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/0f47e88e8afd4d28987038cdfdf0e663 2024-12-03T04:44:04,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/10f743bd51c24ce482de30e06552fca1 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/10f743bd51c24ce482de30e06552fca1 2024-12-03T04:44:04,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/706b471ccd9145e4a7b5b975658986b2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/8373ac837db746fd99d76ab0422650b3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/8373c9ea02d0411dbca86d6795834362, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b35b7e8074e14011aa7eada15840a773, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/de5fc0b3639e47d4bc72c8f04549d896, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/60b699fd087940a589b9da953b4ef333, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/952ce47bdb4a4f7abfa4b51ce555248a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/49a9fc58d9a449ed9dce1f7e657567fe, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6ff55132b2554947b660ba85070d3242, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/ec4d2c1b3fe2431b8bcf0ff4fab1752d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/20ec4163d88b43b0a68f1739cf0b57c5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/071bb2e4f8a44e70a14d15b3657c85bc, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b098e10bc92e49d9a37e9b14a20a01f4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/1e2ce25292f84c4f913d21593d66f5e1, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/62da020adf344a50ab911ce8e7e3af05, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b78cc94adf3e4ad79dbc4b9b48b3dc33, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/88c9f3cb0951422f922497a210f2c700, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/5d04206f7c804ea08966961130b285ff, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/a8252349e1bf4473ae67d6d99f9a848f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6460be02655f4963bc74ed1e0e341b5b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/a55496585eb34777bacc976157eb771a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/3680d51c0e0148dba4981c97eaa8e19b] to archive 2024-12-03T04:44:04,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:44:04,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/706b471ccd9145e4a7b5b975658986b2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/706b471ccd9145e4a7b5b975658986b2 2024-12-03T04:44:04,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/8373ac837db746fd99d76ab0422650b3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/8373ac837db746fd99d76ab0422650b3 2024-12-03T04:44:04,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/8373c9ea02d0411dbca86d6795834362 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/8373c9ea02d0411dbca86d6795834362 2024-12-03T04:44:04,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b35b7e8074e14011aa7eada15840a773 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b35b7e8074e14011aa7eada15840a773 2024-12-03T04:44:04,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/de5fc0b3639e47d4bc72c8f04549d896 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/de5fc0b3639e47d4bc72c8f04549d896 2024-12-03T04:44:04,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/60b699fd087940a589b9da953b4ef333 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/60b699fd087940a589b9da953b4ef333 2024-12-03T04:44:04,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/952ce47bdb4a4f7abfa4b51ce555248a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/952ce47bdb4a4f7abfa4b51ce555248a 2024-12-03T04:44:04,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/49a9fc58d9a449ed9dce1f7e657567fe to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/49a9fc58d9a449ed9dce1f7e657567fe 2024-12-03T04:44:04,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6ff55132b2554947b660ba85070d3242 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6ff55132b2554947b660ba85070d3242 2024-12-03T04:44:04,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/ec4d2c1b3fe2431b8bcf0ff4fab1752d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/ec4d2c1b3fe2431b8bcf0ff4fab1752d 2024-12-03T04:44:04,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/20ec4163d88b43b0a68f1739cf0b57c5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/20ec4163d88b43b0a68f1739cf0b57c5 2024-12-03T04:44:04,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/071bb2e4f8a44e70a14d15b3657c85bc to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/071bb2e4f8a44e70a14d15b3657c85bc 2024-12-03T04:44:04,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b098e10bc92e49d9a37e9b14a20a01f4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b098e10bc92e49d9a37e9b14a20a01f4 2024-12-03T04:44:04,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/1e2ce25292f84c4f913d21593d66f5e1 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/1e2ce25292f84c4f913d21593d66f5e1 2024-12-03T04:44:04,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/62da020adf344a50ab911ce8e7e3af05 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/62da020adf344a50ab911ce8e7e3af05 2024-12-03T04:44:04,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b78cc94adf3e4ad79dbc4b9b48b3dc33 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b78cc94adf3e4ad79dbc4b9b48b3dc33 2024-12-03T04:44:04,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/88c9f3cb0951422f922497a210f2c700 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/88c9f3cb0951422f922497a210f2c700 2024-12-03T04:44:04,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/5d04206f7c804ea08966961130b285ff to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/5d04206f7c804ea08966961130b285ff 2024-12-03T04:44:04,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/a8252349e1bf4473ae67d6d99f9a848f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/a8252349e1bf4473ae67d6d99f9a848f 2024-12-03T04:44:04,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6460be02655f4963bc74ed1e0e341b5b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/6460be02655f4963bc74ed1e0e341b5b 2024-12-03T04:44:04,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/a55496585eb34777bacc976157eb771a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/a55496585eb34777bacc976157eb771a 2024-12-03T04:44:04,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/3680d51c0e0148dba4981c97eaa8e19b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/3680d51c0e0148dba4981c97eaa8e19b 2024-12-03T04:44:04,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/39e391ed7bd34782ac7bee403943bdc2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/09fa721d3bb341619b6e77a5cfcb1f9c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/cc7cd9ee9fa94cc1a8e5033412eda0bb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/eeed359fdd56470ba74ae3ad893cc58c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/189ff607e20a4f2a9bf0f86750eda18d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/a09803a4d063444ca2de68ac2509a600, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f99303e4cddc4312832626d3ead492c1, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/58e1ff60307f4841a2e470df9ec24df3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/16f37375ae404d408a2a30beef51fae3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/e16bb56e376947bd90f3007d40d32358, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/d46e47b2bab14f3abe97bda0807b2906, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ae93a879701a44ebab495b71fdd118fd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/891b828b84ea4e1e8cdd687a868e7f42, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/c52dc0c9112b481a9a1125afcfeee47b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/781937d2da1245758c025b700c23b83f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/af7553a201684facac4e8115211c3649, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/3303a26b245f48f3bd79f6c2a0ccba13, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ebd73f14a6664c3c98bec45ebd12338e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/5f0a9814a9ca49a0b8f4900c98338eb4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/dff0195cd0374561ade01a6ed2c777f5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f1dbfd4a8f114d579003f0a006bd97b7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/4ba03aae61c044f6b631532961c79faa] to archive 2024-12-03T04:44:04,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:44:04,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/39e391ed7bd34782ac7bee403943bdc2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/39e391ed7bd34782ac7bee403943bdc2 2024-12-03T04:44:04,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/09fa721d3bb341619b6e77a5cfcb1f9c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/09fa721d3bb341619b6e77a5cfcb1f9c 2024-12-03T04:44:04,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/cc7cd9ee9fa94cc1a8e5033412eda0bb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/cc7cd9ee9fa94cc1a8e5033412eda0bb 2024-12-03T04:44:04,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/eeed359fdd56470ba74ae3ad893cc58c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/eeed359fdd56470ba74ae3ad893cc58c 2024-12-03T04:44:04,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/189ff607e20a4f2a9bf0f86750eda18d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/189ff607e20a4f2a9bf0f86750eda18d 2024-12-03T04:44:04,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/a09803a4d063444ca2de68ac2509a600 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/a09803a4d063444ca2de68ac2509a600 2024-12-03T04:44:04,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f99303e4cddc4312832626d3ead492c1 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f99303e4cddc4312832626d3ead492c1 2024-12-03T04:44:04,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/58e1ff60307f4841a2e470df9ec24df3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/58e1ff60307f4841a2e470df9ec24df3 2024-12-03T04:44:04,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/16f37375ae404d408a2a30beef51fae3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/16f37375ae404d408a2a30beef51fae3 2024-12-03T04:44:04,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/e16bb56e376947bd90f3007d40d32358 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/e16bb56e376947bd90f3007d40d32358 2024-12-03T04:44:04,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/d46e47b2bab14f3abe97bda0807b2906 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/d46e47b2bab14f3abe97bda0807b2906 2024-12-03T04:44:04,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ae93a879701a44ebab495b71fdd118fd to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ae93a879701a44ebab495b71fdd118fd 2024-12-03T04:44:04,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/891b828b84ea4e1e8cdd687a868e7f42 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/891b828b84ea4e1e8cdd687a868e7f42 2024-12-03T04:44:04,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/c52dc0c9112b481a9a1125afcfeee47b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/c52dc0c9112b481a9a1125afcfeee47b 2024-12-03T04:44:04,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/781937d2da1245758c025b700c23b83f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/781937d2da1245758c025b700c23b83f 2024-12-03T04:44:04,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/af7553a201684facac4e8115211c3649 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/af7553a201684facac4e8115211c3649 2024-12-03T04:44:04,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/3303a26b245f48f3bd79f6c2a0ccba13 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/3303a26b245f48f3bd79f6c2a0ccba13 2024-12-03T04:44:04,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ebd73f14a6664c3c98bec45ebd12338e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/ebd73f14a6664c3c98bec45ebd12338e 2024-12-03T04:44:04,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/5f0a9814a9ca49a0b8f4900c98338eb4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/5f0a9814a9ca49a0b8f4900c98338eb4 2024-12-03T04:44:04,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/dff0195cd0374561ade01a6ed2c777f5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/dff0195cd0374561ade01a6ed2c777f5 2024-12-03T04:44:04,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f1dbfd4a8f114d579003f0a006bd97b7 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/f1dbfd4a8f114d579003f0a006bd97b7 2024-12-03T04:44:04,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/4ba03aae61c044f6b631532961c79faa to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/4ba03aae61c044f6b631532961c79faa 2024-12-03T04:44:04,970 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/recovered.edits/397.seqid, newMaxSeqId=397, maxSeqId=4 2024-12-03T04:44:04,971 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. 2024-12-03T04:44:04,971 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for ad1a6acd12c06a97ea4048034fea205b: Waiting for close lock at 1733201043040Running coprocessor pre-close hooks at 1733201043040Disabling compacts and flushes for region at 1733201043040Disabling writes for close at 1733201043040Obtaining lock to block concurrent updates at 1733201043041 (+1 ms)Preparing flush snapshotting stores in ad1a6acd12c06a97ea4048034fea205b at 1733201043041Finished memstore snapshotting TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733201043041Flushing stores of TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b. at 1733201043042 (+1 ms)Flushing ad1a6acd12c06a97ea4048034fea205b/A: creating writer at 1733201043042Flushing mob file ad1a6acd12c06a97ea4048034fea205b/A: appending metadata at 1733201043075 (+33 ms)Flushing mob file ad1a6acd12c06a97ea4048034fea205b/A: closing flushed file at 1733201043075Flushing ad1a6acd12c06a97ea4048034fea205b/A: appending metadata at 1733201043514 (+439 ms)Flushing ad1a6acd12c06a97ea4048034fea205b/A: closing flushed file at 1733201043514Flushing ad1a6acd12c06a97ea4048034fea205b/B: creating writer at 1733201043937 (+423 ms)Flushing ad1a6acd12c06a97ea4048034fea205b/B: appending metadata at 1733201043940 (+3 ms)Flushing ad1a6acd12c06a97ea4048034fea205b/B: closing flushed file at 1733201043940Flushing ad1a6acd12c06a97ea4048034fea205b/C: creating writer at 1733201044361 (+421 ms)Flushing ad1a6acd12c06a97ea4048034fea205b/C: appending metadata at 1733201044365 (+4 ms)Flushing ad1a6acd12c06a97ea4048034fea205b/C: closing flushed file at 1733201044365Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a929454: reopening flushed file at 1733201044787 (+422 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5428781a: reopening flushed file at 1733201044797 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74577e58: reopening flushed file at 1733201044804 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ad1a6acd12c06a97ea4048034fea205b in 1770ms, sequenceid=394, compaction requested=true at 1733201044810 (+6 ms)Writing region close event to WAL at 1733201044963 (+153 ms)Running coprocessor post-close hooks at 1733201044970 (+7 ms)Closed at 1733201044971 (+1 ms) 2024-12-03T04:44:04,975 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:04,975 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=ad1a6acd12c06a97ea4048034fea205b, regionState=CLOSED 2024-12-03T04:44:04,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:44:04,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-03T04:44:04,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure ad1a6acd12c06a97ea4048034fea205b, server=32beccd76b35,43193,1733200986730 in 2.0910 sec 2024-12-03T04:44:04,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-12-03T04:44:04,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ad1a6acd12c06a97ea4048034fea205b, UNASSIGN in 2.0970 sec 2024-12-03T04:44:04,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-03T04:44:04,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1020 sec 2024-12-03T04:44:04,987 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201044987"}]},"ts":"1733201044987"} 2024-12-03T04:44:04,988 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T04:44:04,988 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T04:44:04,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1140 sec 2024-12-03T04:44:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T04:44:05,032 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:05,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T04:44:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:05,034 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:05,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T04:44:05,035 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:05,038 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,040 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/recovered.edits] 2024-12-03T04:44:05,043 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2817e6fccb1240988dc19331c2291ae4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2817e6fccb1240988dc19331c2291ae4 2024-12-03T04:44:05,044 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2a43947329524d0ea441355ff3e3d11c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/2a43947329524d0ea441355ff3e3d11c 2024-12-03T04:44:05,046 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/b3016785667443cca469a1be8328e0d5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/A/b3016785667443cca469a1be8328e0d5 2024-12-03T04:44:05,049 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/1370404ac36d4e80971a489c977784b1 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/1370404ac36d4e80971a489c977784b1 2024-12-03T04:44:05,054 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/4f8b52aa43aa4b0db3ae09f7b733e903 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/4f8b52aa43aa4b0db3ae09f7b733e903 2024-12-03T04:44:05,056 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b4575ab32d7741398b25c54b2f2c7968 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/B/b4575ab32d7741398b25c54b2f2c7968 2024-12-03T04:44:05,059 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/50fdad212d264a6491d5b43ebe97922e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/50fdad212d264a6491d5b43ebe97922e 2024-12-03T04:44:05,060 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/7bdcafa8d223475ca71b5d86762f4726 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/7bdcafa8d223475ca71b5d86762f4726 2024-12-03T04:44:05,061 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/d3df8776c2e94270955e08fe50bce197 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/C/d3df8776c2e94270955e08fe50bce197 2024-12-03T04:44:05,064 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/recovered.edits/397.seqid to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b/recovered.edits/397.seqid 2024-12-03T04:44:05,064 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,064 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T04:44:05,065 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T04:44:05,066 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T04:44:05,074 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031663b8b95df542f9af5fd5acb9f11a6a_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031663b8b95df542f9af5fd5acb9f11a6a_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,075 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120322bcb1f0aaff4031a72296f3ea77b9e9_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120322bcb1f0aaff4031a72296f3ea77b9e9_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,078 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032b22b9c9006a4f09a8e880ab9679a2b2_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032b22b9c9006a4f09a8e880ab9679a2b2_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,079 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120333d0f6a5655b4e52b2e52b3fc1defdfe_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120333d0f6a5655b4e52b2e52b3fc1defdfe_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,081 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033f4d3edce65e42e386844c92b0bb55b4_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033f4d3edce65e42e386844c92b0bb55b4_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,083 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035b971e9ee0c2457e8bb667bcbeb57914_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035b971e9ee0c2457e8bb667bcbeb57914_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,084 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036206a33c4b354113827c91d9e7a6f769_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036206a33c4b354113827c91d9e7a6f769_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,086 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037c80698c279943b08a1d45b85c6b420a_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037c80698c279943b08a1d45b85c6b420a_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,087 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120387f12bc08c964cb3b37fd4cc679a171a_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120387f12bc08c964cb3b37fd4cc679a171a_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,089 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203881e408f709544779ee41850fd574e5b_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203881e408f709544779ee41850fd574e5b_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,090 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038df5e1d224624b4693ab56042d835fd1_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038df5e1d224624b4693ab56042d835fd1_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,092 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b0bd3ac03f944668beb71d78f3ba0e18_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b0bd3ac03f944668beb71d78f3ba0e18_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,094 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b6946c65cb4844359b1f5c42289c89d6_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b6946c65cb4844359b1f5c42289c89d6_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,095 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b8c6ab4610e141159b6c0c60f6004fb4_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b8c6ab4610e141159b6c0c60f6004fb4_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,096 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cb0fbb2ee70a4888b6e83d81fa02a1a1_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cb0fbb2ee70a4888b6e83d81fa02a1a1_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,097 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d3e064e0403e4980afc9097bbdc507d3_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d3e064e0403e4980afc9097bbdc507d3_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,098 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d760c211c53c41108add100732645084_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d760c211c53c41108add100732645084_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,099 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ec4cef999794446c8a00f50025e5c06d_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ec4cef999794446c8a00f50025e5c06d_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,100 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f3eb36c298af4ae59e48788e89f473c3_ad1a6acd12c06a97ea4048034fea205b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f3eb36c298af4ae59e48788e89f473c3_ad1a6acd12c06a97ea4048034fea205b 2024-12-03T04:44:05,100 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T04:44:05,103 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:05,105 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T04:44:05,107 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T04:44:05,108 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:05,108 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T04:44:05,108 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733201045108"}]},"ts":"9223372036854775807"} 2024-12-03T04:44:05,110 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T04:44:05,110 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ad1a6acd12c06a97ea4048034fea205b, NAME => 'TestAcidGuarantees,,1733201016207.ad1a6acd12c06a97ea4048034fea205b.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T04:44:05,110 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T04:44:05,110 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733201045110"}]},"ts":"9223372036854775807"} 2024-12-03T04:44:05,111 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T04:44:05,112 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:05,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-12-03T04:44:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T04:44:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T04:44:05,142 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:05,151 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247 (was 247), OpenFileDescriptor=455 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=746 (was 569) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3630 (was 4904) 2024-12-03T04:44:05,159 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=746, ProcessCount=11, AvailableMemoryMB=3630 2024-12-03T04:44:05,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T04:44:05,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:44:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:05,162 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T04:44:05,163 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:05,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-12-03T04:44:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T04:44:05,163 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T04:44:05,163 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T04:44:05,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742042_1218 (size=1061) 2024-12-03T04:44:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T04:44:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T04:44:05,571 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676 2024-12-03T04:44:05,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742043_1219 (size=53) 2024-12-03T04:44:05,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:44:05,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c65c1da21ae3e3e43d67b7de8f358916, disabling compactions & flushes 2024-12-03T04:44:05,615 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:05,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:05,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. after waiting 0 ms 2024-12-03T04:44:05,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:05,615 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:05,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c65c1da21ae3e3e43d67b7de8f358916: Waiting for close lock at 1733201045615Disabling compacts and flushes for region at 1733201045615Disabling writes for close at 1733201045615Writing region close event to WAL at 1733201045615Closed at 1733201045615 2024-12-03T04:44:05,624 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T04:44:05,624 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733201045624"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733201045624"}]},"ts":"1733201045624"} 2024-12-03T04:44:05,626 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T04:44:05,629 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T04:44:05,630 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201045629"}]},"ts":"1733201045629"} 2024-12-03T04:44:05,631 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T04:44:05,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c65c1da21ae3e3e43d67b7de8f358916, ASSIGN}] 2024-12-03T04:44:05,633 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c65c1da21ae3e3e43d67b7de8f358916, ASSIGN 2024-12-03T04:44:05,634 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c65c1da21ae3e3e43d67b7de8f358916, ASSIGN; state=OFFLINE, location=32beccd76b35,43193,1733200986730; forceNewPlan=false, retain=false 2024-12-03T04:44:05,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=c65c1da21ae3e3e43d67b7de8f358916, regionState=OPENING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:44:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T04:44:05,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c65c1da21ae3e3e43d67b7de8f358916, ASSIGN because future has completed 2024-12-03T04:44:05,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:44:05,966 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:05,966 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:44:05,967 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:05,967 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:44:05,967 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:05,967 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:05,969 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:05,970 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:44:05,973 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c65c1da21ae3e3e43d67b7de8f358916 columnFamilyName A 2024-12-03T04:44:05,973 DEBUG [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:05,974 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] regionserver.HStore(327): Store=c65c1da21ae3e3e43d67b7de8f358916/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:44:05,974 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:05,978 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:44:05,978 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c65c1da21ae3e3e43d67b7de8f358916 columnFamilyName B 2024-12-03T04:44:05,978 DEBUG [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:05,980 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] regionserver.HStore(327): Store=c65c1da21ae3e3e43d67b7de8f358916/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:44:05,980 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:05,985 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:44:05,986 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c65c1da21ae3e3e43d67b7de8f358916 columnFamilyName C 2024-12-03T04:44:05,986 DEBUG [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:05,993 INFO [StoreOpener-c65c1da21ae3e3e43d67b7de8f358916-1 {}] regionserver.HStore(327): Store=c65c1da21ae3e3e43d67b7de8f358916/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:44:05,993 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:05,993 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:05,998 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:06,003 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:06,006 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:06,006 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:06,006 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:44:06,008 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:06,017 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T04:44:06,020 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened c65c1da21ae3e3e43d67b7de8f358916; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64641739, jitterRate=-0.0367630273103714}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:44:06,020 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:06,021 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for c65c1da21ae3e3e43d67b7de8f358916: Running coprocessor pre-open hook at 1733201045967Writing region info on filesystem at 1733201045967Initializing all the Stores at 1733201045968 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201045968Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201045968Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201045968Cleaning up temporary data from old regions at 1733201046006 (+38 ms)Running coprocessor post-open hooks at 1733201046020 (+14 ms)Region opened successfully at 1733201046021 (+1 ms) 2024-12-03T04:44:06,023 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., pid=57, masterSystemTime=1733201045962 2024-12-03T04:44:06,026 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=c65c1da21ae3e3e43d67b7de8f358916, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,027 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:06,027 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:06,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:44:06,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-03T04:44:06,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 in 233 msec 2024-12-03T04:44:06,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-03T04:44:06,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c65c1da21ae3e3e43d67b7de8f358916, ASSIGN in 405 msec 2024-12-03T04:44:06,044 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T04:44:06,044 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201046044"}]},"ts":"1733201046044"} 2024-12-03T04:44:06,045 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T04:44:06,046 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T04:44:06,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 885 msec 2024-12-03T04:44:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T04:44:06,301 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:06,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:06,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:06,304 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f490696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:06,305 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15302126, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,306 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:06,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:06,309 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T04:44:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,38397,1733200986071 2024-12-03T04:44:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@437944a6 2024-12-03T04:44:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T04:44:06,310 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T04:44:06,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:06,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:06,312 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:06,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:06,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab6cd6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:06,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:06,313 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:06,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ba6db4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:06,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:06,315 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb5b38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:06,316 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c67f2c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:06,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:06,318 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18139fc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:06,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:06,319 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16452675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:06,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:06,321 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74da56d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:06,322 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:06,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36070a15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:06,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:06,324 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d32489d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:06,325 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2d4ee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:06,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:06,326 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740509df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:06,328 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@418e1fc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:06,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:06,330 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea932c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:06,332 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:06,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64303841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:06,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:06,334 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78019144, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:06,335 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:06,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cde10b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:06,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:06,337 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@268b7f8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:06,339 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7289f657, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:06,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:06,341 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc04334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:06,343 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ed10d9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:06,353 DEBUG [Thread-1037 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:06,353 DEBUG [Thread-1041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:06,358 DEBUG [Thread-1041 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:06,358 DEBUG [Thread-1041 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:06,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-03T04:44:06,362 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:06,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T04:44:06,364 DEBUG [Thread-1046 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:06,364 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:06,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:06,365 DEBUG [Thread-1050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:44:06,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:06,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:06,368 DEBUG [Thread-1044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:06,369 DEBUG [Thread-1033 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:06,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:06,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:06,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:44:06,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:44:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:44:06,378 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:06,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:44:06,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:44:06,379 DEBUG [Thread-1052 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:06,379 DEBUG [Thread-1052 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:06,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:44:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:06,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:44:06,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:06,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:06,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:06,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:06,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:06,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:06,394 DEBUG [Thread-1048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:06,395 DEBUG [Thread-1048 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:06,395 DEBUG [Thread-1048 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:06,396 DEBUG [Thread-1035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:06,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:06,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:44:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:06,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:06,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:44:06,410 DEBUG [Thread-1039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:06,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:06,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:44:06,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201046403, max timestamp=1733201046434Num uniques 859; 2024-12-03T04:44:06,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:06,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201046403, max timestamp=1733201046434Num uniques 859; 2024-12-03T04:44:06,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:06,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201046403, max timestamp=1733201046434Num uniques 859; 2024-12-03T04:44:06,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:06,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/a2bf9a66db8e4531b206742a32f88fa5 is 50, key is test_row_0/A:col10/1733201046385/Put/seqid=0 2024-12-03T04:44:06,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742044_1220 (size=9657) 2024-12-03T04:44:06,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/a2bf9a66db8e4531b206742a32f88fa5 2024-12-03T04:44:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T04:44:06,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201106494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201106502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733201046484, max timestamp=1733201046507Num uniques 1018; 2024-12-03T04:44:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T04:44:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733201046484, max timestamp=1733201046507Num uniques 1018; 2024-12-03T04:44:06,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T04:44:06,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733201046484, max timestamp=1733201046507Num uniques 1018; 2024-12-03T04:44:06,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T04:44:06,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:06,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:06,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:06,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:06,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:06,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:06,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/1a38e45419594b3c93fc82a03c3d290f is 50, key is test_row_0/B:col10/1733201046385/Put/seqid=0 2024-12-03T04:44:06,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742045_1221 (size=9657) 2024-12-03T04:44:06,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/1a38e45419594b3c93fc82a03c3d290f 2024-12-03T04:44:06,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201106610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201106611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201106618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201106621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201106621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/8e3ebaae08a44a3e87406722bf5c96d1 is 50, key is test_row_0/C:col10/1733201046385/Put/seqid=0 2024-12-03T04:44:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T04:44:06,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742046_1222 (size=9657) 2024-12-03T04:44:06,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:06,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:06,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:06,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:06,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:06,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:06,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201106721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201106722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201106732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201106833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:06,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:06,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:06,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:06,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:06,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:06,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201106844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201106931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201106942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201106942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T04:44:06,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:06,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:06,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:06,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:06,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:06,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/8e3ebaae08a44a3e87406722bf5c96d1 2024-12-03T04:44:07,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/a2bf9a66db8e4531b206742a32f88fa5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/a2bf9a66db8e4531b206742a32f88fa5 2024-12-03T04:44:07,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/a2bf9a66db8e4531b206742a32f88fa5, entries=100, sequenceid=11, filesize=9.4 K 2024-12-03T04:44:07,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/1a38e45419594b3c93fc82a03c3d290f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/1a38e45419594b3c93fc82a03c3d290f 2024-12-03T04:44:07,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/1a38e45419594b3c93fc82a03c3d290f, entries=100, sequenceid=11, filesize=9.4 K 2024-12-03T04:44:07,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/8e3ebaae08a44a3e87406722bf5c96d1 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/8e3ebaae08a44a3e87406722bf5c96d1 2024-12-03T04:44:07,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/8e3ebaae08a44a3e87406722bf5c96d1, entries=100, sequenceid=11, filesize=9.4 K 2024-12-03T04:44:07,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c65c1da21ae3e3e43d67b7de8f358916 in 720ms, sequenceid=11, compaction requested=false 2024-12-03T04:44:07,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:07,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=140.89 KB heapSize=404.77 KB 2024-12-03T04:44:07,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:07,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:44:07,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:07,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:44:07,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:07,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:44:07,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:07,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:07,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d83e8e1fc6e84c2d95d8afb2d304b430 is 50, key is test_row_0/A:col10/1733201046607/Put/seqid=0 2024-12-03T04:44:07,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742047_1223 (size=14341) 2024-12-03T04:44:07,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201107165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201107169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201107242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201107251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201107251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201107282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201107282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:07,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:07,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201107492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201107492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T04:44:07,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d83e8e1fc6e84c2d95d8afb2d304b430 2024-12-03T04:44:07,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/661efa37cca14de8893d31d92a60c4d2 is 50, key is test_row_0/B:col10/1733201046607/Put/seqid=0 2024-12-03T04:44:07,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742048_1224 (size=12001) 2024-12-03T04:44:07,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/661efa37cca14de8893d31d92a60c4d2 2024-12-03T04:44:07,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/be648e8702df4ed38da7d68cd9d19435 is 50, key is test_row_0/C:col10/1733201046607/Put/seqid=0 2024-12-03T04:44:07,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742049_1225 (size=12001) 2024-12-03T04:44:07,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/be648e8702df4ed38da7d68cd9d19435 2024-12-03T04:44:07,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d83e8e1fc6e84c2d95d8afb2d304b430 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d83e8e1fc6e84c2d95d8afb2d304b430 2024-12-03T04:44:07,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d83e8e1fc6e84c2d95d8afb2d304b430, entries=200, sequenceid=66, filesize=14.0 K 2024-12-03T04:44:07,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/661efa37cca14de8893d31d92a60c4d2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/661efa37cca14de8893d31d92a60c4d2 2024-12-03T04:44:07,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/661efa37cca14de8893d31d92a60c4d2, entries=150, sequenceid=66, filesize=11.7 K 2024-12-03T04:44:07,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/be648e8702df4ed38da7d68cd9d19435 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/be648e8702df4ed38da7d68cd9d19435 2024-12-03T04:44:07,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/be648e8702df4ed38da7d68cd9d19435, entries=150, sequenceid=66, filesize=11.7 K 2024-12-03T04:44:07,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~404.72 KB/414432, currentSize=40.25 KB/41220 for c65c1da21ae3e3e43d67b7de8f358916 in 567ms, sequenceid=66, compaction requested=false 2024-12-03T04:44:07,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:07,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:44:07,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:07,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:07,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:07,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:07,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:07,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:07,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:07,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:07,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/7c87d7ade0394d0ab06b6ff752e1154f is 50, key is test_row_0/A:col10/1733201047761/Put/seqid=0 2024-12-03T04:44:07,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742050_1226 (size=12001) 2024-12-03T04:44:07,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201107816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201047802, max timestamp=1733201047821Num uniques 859; 2024-12-03T04:44:07,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:44:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201047802, max timestamp=1733201047821Num uniques 859; 2024-12-03T04:44:07,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:07,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733201047802, max timestamp=1733201047821Num uniques 859; 2024-12-03T04:44:07,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T04:44:07,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201107861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201107871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201107873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201107874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:07,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201107931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201107982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201107982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201107991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201107991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:08,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:08,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:08,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:08,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:08,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:08,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201108141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201108192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201108192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/7c87d7ade0394d0ab06b6ff752e1154f 2024-12-03T04:44:08,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201108201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201108201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/fa0286b2e31d4ab5b5afcb5cb9780188 is 50, key is test_row_0/B:col10/1733201047761/Put/seqid=0 2024-12-03T04:44:08,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:08,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:08,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742051_1227 (size=12001) 2024-12-03T04:44:08,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/fa0286b2e31d4ab5b5afcb5cb9780188 2024-12-03T04:44:08,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/23e7944239784b839b674d5b69380b21 is 50, key is test_row_0/C:col10/1733201047761/Put/seqid=0 2024-12-03T04:44:08,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742052_1228 (size=12001) 2024-12-03T04:44:08,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/23e7944239784b839b674d5b69380b21 2024-12-03T04:44:08,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/7c87d7ade0394d0ab06b6ff752e1154f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/7c87d7ade0394d0ab06b6ff752e1154f 2024-12-03T04:44:08,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/7c87d7ade0394d0ab06b6ff752e1154f, entries=150, sequenceid=77, filesize=11.7 K 2024-12-03T04:44:08,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/fa0286b2e31d4ab5b5afcb5cb9780188 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/fa0286b2e31d4ab5b5afcb5cb9780188 2024-12-03T04:44:08,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/fa0286b2e31d4ab5b5afcb5cb9780188, entries=150, sequenceid=77, filesize=11.7 K 2024-12-03T04:44:08,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/23e7944239784b839b674d5b69380b21 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/23e7944239784b839b674d5b69380b21 2024-12-03T04:44:08,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/23e7944239784b839b674d5b69380b21, entries=150, sequenceid=77, filesize=11.7 K 2024-12-03T04:44:08,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.02 KB/139285 for c65c1da21ae3e3e43d67b7de8f358916 in 526ms, sequenceid=77, compaction requested=true 2024-12-03T04:44:08,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:08,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:08,288 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:08,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:08,288 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:08,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:08,289 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:08,289 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/A is initiating minor compaction (all files) 2024-12-03T04:44:08,289 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/A in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:08,289 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/a2bf9a66db8e4531b206742a32f88fa5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d83e8e1fc6e84c2d95d8afb2d304b430, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/7c87d7ade0394d0ab06b6ff752e1154f] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=35.2 K 2024-12-03T04:44:08,290 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:08,290 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/B is initiating minor compaction (all files) 2024-12-03T04:44:08,290 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/B in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:08,290 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/1a38e45419594b3c93fc82a03c3d290f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/661efa37cca14de8893d31d92a60c4d2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/fa0286b2e31d4ab5b5afcb5cb9780188] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=32.9 K 2024-12-03T04:44:08,290 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2bf9a66db8e4531b206742a32f88fa5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733201046381 2024-12-03T04:44:08,291 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a38e45419594b3c93fc82a03c3d290f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733201046381 2024-12-03T04:44:08,291 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting d83e8e1fc6e84c2d95d8afb2d304b430, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733201046599 2024-12-03T04:44:08,291 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 661efa37cca14de8893d31d92a60c4d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733201046599 2024-12-03T04:44:08,291 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c87d7ade0394d0ab06b6ff752e1154f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733201047161 2024-12-03T04:44:08,293 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting fa0286b2e31d4ab5b5afcb5cb9780188, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733201047161 2024-12-03T04:44:08,307 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#A#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:08,307 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d599791007cc4f1ebea4d15400fb8b5b is 50, key is test_row_0/A:col10/1733201047761/Put/seqid=0 2024-12-03T04:44:08,308 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:08,312 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/c943a113772744c6951cfe2008353f96 is 50, key is test_row_0/B:col10/1733201047761/Put/seqid=0 2024-12-03T04:44:08,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742053_1229 (size=12104) 2024-12-03T04:44:08,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742054_1230 (size=12104) 2024-12-03T04:44:08,328 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d599791007cc4f1ebea4d15400fb8b5b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d599791007cc4f1ebea4d15400fb8b5b 2024-12-03T04:44:08,336 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/A of c65c1da21ae3e3e43d67b7de8f358916 into d599791007cc4f1ebea4d15400fb8b5b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:08,336 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:08,336 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/A, priority=13, startTime=1733201048288; duration=0sec 2024-12-03T04:44:08,336 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:08,337 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:A 2024-12-03T04:44:08,337 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:08,340 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:08,340 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/C is initiating minor compaction (all files) 2024-12-03T04:44:08,340 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/C in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:08,340 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/8e3ebaae08a44a3e87406722bf5c96d1, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/be648e8702df4ed38da7d68cd9d19435, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/23e7944239784b839b674d5b69380b21] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=32.9 K 2024-12-03T04:44:08,341 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e3ebaae08a44a3e87406722bf5c96d1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733201046381 2024-12-03T04:44:08,341 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting be648e8702df4ed38da7d68cd9d19435, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733201046599 2024-12-03T04:44:08,342 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23e7944239784b839b674d5b69380b21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733201047161 2024-12-03T04:44:08,364 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:08,364 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/04d3046fb7d24cb4814f7a95dadd325f is 50, key is test_row_0/C:col10/1733201047761/Put/seqid=0 2024-12-03T04:44:08,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742055_1231 (size=12104) 2024-12-03T04:44:08,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T04:44:08,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:08,387 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=136.02 KB heapSize=389.15 KB 2024-12-03T04:44:08,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:08,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:08,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:08,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:08,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:08,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:08,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f645df993c7e45debb503bb0814edab8 is 50, key is test_row_0/A:col10/1733201047872/Put/seqid=0 2024-12-03T04:44:08,397 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/04d3046fb7d24cb4814f7a95dadd325f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/04d3046fb7d24cb4814f7a95dadd325f 2024-12-03T04:44:08,403 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/C of c65c1da21ae3e3e43d67b7de8f358916 into 04d3046fb7d24cb4814f7a95dadd325f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:08,403 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:08,403 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/C, priority=13, startTime=1733201048289; duration=0sec 2024-12-03T04:44:08,404 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:08,404 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:C 2024-12-03T04:44:08,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742056_1232 (size=12001) 2024-12-03T04:44:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:08,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:08,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201108503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201108504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201108508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201108515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201108515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T04:44:08,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201108621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:08,736 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/c943a113772744c6951cfe2008353f96 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/c943a113772744c6951cfe2008353f96 2024-12-03T04:44:08,743 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/B of c65c1da21ae3e3e43d67b7de8f358916 into c943a113772744c6951cfe2008353f96(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:08,743 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:08,743 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/B, priority=13, startTime=1733201048288; duration=0sec 2024-12-03T04:44:08,743 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:08,743 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:B 2024-12-03T04:44:08,813 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f645df993c7e45debb503bb0814edab8 2024-12-03T04:44:08,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/2359705d157247818bdf8f008b61374d is 50, key is test_row_0/B:col10/1733201047872/Put/seqid=0 2024-12-03T04:44:08,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742057_1233 (size=12001) 2024-12-03T04:44:08,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201108831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:08,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:08,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:09,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201109014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:09,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201109021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:09,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201109031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201109033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:09,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201109142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:09,225 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/2359705d157247818bdf8f008b61374d 2024-12-03T04:44:09,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/6cdd852b8f494c72a32c7fbdf3268997 is 50, key is test_row_0/C:col10/1733201047872/Put/seqid=0 2024-12-03T04:44:09,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742058_1234 (size=12001) 2024-12-03T04:44:09,268 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/6cdd852b8f494c72a32c7fbdf3268997 2024-12-03T04:44:09,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f645df993c7e45debb503bb0814edab8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f645df993c7e45debb503bb0814edab8 2024-12-03T04:44:09,295 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f645df993c7e45debb503bb0814edab8, entries=150, sequenceid=115, filesize=11.7 K 2024-12-03T04:44:09,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/2359705d157247818bdf8f008b61374d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/2359705d157247818bdf8f008b61374d 2024-12-03T04:44:09,304 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/2359705d157247818bdf8f008b61374d, entries=150, sequenceid=115, filesize=11.7 K 2024-12-03T04:44:09,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/6cdd852b8f494c72a32c7fbdf3268997 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/6cdd852b8f494c72a32c7fbdf3268997 2024-12-03T04:44:09,314 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/6cdd852b8f494c72a32c7fbdf3268997, entries=150, sequenceid=115, filesize=11.7 K 2024-12-03T04:44:09,317 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~136.02 KB/139285, heapSize ~389.10 KB/398440, currentSize=46.96 KB/48090 for c65c1da21ae3e3e43d67b7de8f358916 in 931ms, sequenceid=115, compaction requested=false 2024-12-03T04:44:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-03T04:44:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-03T04:44:09,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-03T04:44:09,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9550 sec 2024-12-03T04:44:09,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.9620 sec 2024-12-03T04:44:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:09,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T04:44:09,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:09,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:09,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:09,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:09,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:09,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:09,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/598d2b149f744e4481fdaf94c54cd14d is 50, key is test_row_0/A:col10/1733201048469/Put/seqid=0 2024-12-03T04:44:09,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742059_1235 (size=14341) 2024-12-03T04:44:09,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201049751, max timestamp=1733201049805Num uniques 859; 2024-12-03T04:44:09,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201049751, max timestamp=1733201049805Num uniques 859; 2024-12-03T04:44:09,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:09,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:09,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201049751, max timestamp=1733201049805Num uniques 859; 2024-12-03T04:44:09,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:09,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201109928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:09,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:10,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201110034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:10,041 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201110034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:10,042 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:10,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201110041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:10,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201110042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:10,048 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:10,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201110052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:10,058 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:10,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/598d2b149f744e4481fdaf94c54cd14d 2024-12-03T04:44:10,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/b7a44e5b836e4d2797145bd88dfb349b is 50, key is test_row_0/B:col10/1733201048469/Put/seqid=0 2024-12-03T04:44:10,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742060_1236 (size=12001) 2024-12-03T04:44:10,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201110252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:10,532 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T04:44:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T04:44:10,541 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:10,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-03T04:44:10,544 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T04:44:10,545 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:10,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:10,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201110561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T04:44:10,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/b7a44e5b836e4d2797145bd88dfb349b 2024-12-03T04:44:10,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T04:44:10,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/08fe265e54f844579f5eabc02e2bc739 is 50, key is test_row_0/C:col10/1733201048469/Put/seqid=0 2024-12-03T04:44:10,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:10,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:10,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:10,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:10,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:10,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742061_1237 (size=12001) 2024-12-03T04:44:10,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/08fe265e54f844579f5eabc02e2bc739 2024-12-03T04:44:10,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/598d2b149f744e4481fdaf94c54cd14d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/598d2b149f744e4481fdaf94c54cd14d 2024-12-03T04:44:10,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/598d2b149f744e4481fdaf94c54cd14d, entries=200, sequenceid=128, filesize=14.0 K 2024-12-03T04:44:10,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/b7a44e5b836e4d2797145bd88dfb349b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b7a44e5b836e4d2797145bd88dfb349b 2024-12-03T04:44:10,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b7a44e5b836e4d2797145bd88dfb349b, entries=150, sequenceid=128, filesize=11.7 K 2024-12-03T04:44:10,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T04:44:10,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/08fe265e54f844579f5eabc02e2bc739 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/08fe265e54f844579f5eabc02e2bc739 2024-12-03T04:44:10,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:10,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:10,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:10,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:10,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T04:44:10,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/08fe265e54f844579f5eabc02e2bc739, entries=150, sequenceid=128, filesize=11.7 K 2024-12-03T04:44:10,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c65c1da21ae3e3e43d67b7de8f358916 in 1230ms, sequenceid=128, compaction requested=true 2024-12-03T04:44:10,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:10,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:10,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:10,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:10,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:10,885 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:10,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:10,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:10,885 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:10,889 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:10,889 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:10,890 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/A is initiating minor compaction (all files) 2024-12-03T04:44:10,890 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/B is initiating minor compaction (all files) 2024-12-03T04:44:10,890 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/A in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:10,890 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d599791007cc4f1ebea4d15400fb8b5b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f645df993c7e45debb503bb0814edab8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/598d2b149f744e4481fdaf94c54cd14d] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=37.5 K 2024-12-03T04:44:10,890 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting d599791007cc4f1ebea4d15400fb8b5b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733201047161 2024-12-03T04:44:10,891 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/B in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:10,891 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/c943a113772744c6951cfe2008353f96, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/2359705d157247818bdf8f008b61374d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b7a44e5b836e4d2797145bd88dfb349b] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=35.3 K 2024-12-03T04:44:10,892 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f645df993c7e45debb503bb0814edab8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733201047847 2024-12-03T04:44:10,892 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 598d2b149f744e4481fdaf94c54cd14d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733201048458 2024-12-03T04:44:10,893 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c943a113772744c6951cfe2008353f96, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733201047161 2024-12-03T04:44:10,897 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 2359705d157247818bdf8f008b61374d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733201047847 2024-12-03T04:44:10,901 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a44e5b836e4d2797145bd88dfb349b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733201048469 2024-12-03T04:44:10,941 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#B#compaction#195 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:10,942 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/595b7db7797642628bc5a0d187f5f0f1 is 50, key is test_row_0/B:col10/1733201048469/Put/seqid=0 2024-12-03T04:44:10,948 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#A#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:10,948 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/c71c99a7b87243b29f2b594cdb86ea89 is 50, key is test_row_0/A:col10/1733201048469/Put/seqid=0 2024-12-03T04:44:10,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742062_1238 (size=12207) 2024-12-03T04:44:10,972 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/595b7db7797642628bc5a0d187f5f0f1 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/595b7db7797642628bc5a0d187f5f0f1 2024-12-03T04:44:10,978 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/B of c65c1da21ae3e3e43d67b7de8f358916 into 595b7db7797642628bc5a0d187f5f0f1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:10,979 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:10,979 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/B, priority=13, startTime=1733201050885; duration=0sec 2024-12-03T04:44:10,979 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:10,979 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:B 2024-12-03T04:44:10,979 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:10,980 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:10,980 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/C is initiating minor compaction (all files) 2024-12-03T04:44:10,980 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/C in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:10,980 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/04d3046fb7d24cb4814f7a95dadd325f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/6cdd852b8f494c72a32c7fbdf3268997, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/08fe265e54f844579f5eabc02e2bc739] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=35.3 K 2024-12-03T04:44:10,981 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d3046fb7d24cb4814f7a95dadd325f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733201047161 2024-12-03T04:44:10,981 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cdd852b8f494c72a32c7fbdf3268997, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733201047847 2024-12-03T04:44:10,982 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 08fe265e54f844579f5eabc02e2bc739, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733201048469 2024-12-03T04:44:10,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742063_1239 (size=12207) 2024-12-03T04:44:10,999 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/c71c99a7b87243b29f2b594cdb86ea89 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c71c99a7b87243b29f2b594cdb86ea89 2024-12-03T04:44:11,004 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#C#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:11,004 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/92b2fcf2928c41bc82fd5a15d4107a21 is 50, key is test_row_0/C:col10/1733201048469/Put/seqid=0 2024-12-03T04:44:11,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T04:44:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:11,009 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T04:44:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:11,011 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/A of c65c1da21ae3e3e43d67b7de8f358916 into c71c99a7b87243b29f2b594cdb86ea89(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:11,011 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:11,011 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/A, priority=13, startTime=1733201050885; duration=0sec 2024-12-03T04:44:11,012 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:11,012 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:A 2024-12-03T04:44:11,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/23aaf77f729e4746a942d1e5f233449f is 50, key is test_row_0/A:col10/1733201049926/Put/seqid=0 2024-12-03T04:44:11,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742064_1240 (size=12207) 2024-12-03T04:44:11,031 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/92b2fcf2928c41bc82fd5a15d4107a21 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/92b2fcf2928c41bc82fd5a15d4107a21 2024-12-03T04:44:11,037 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/C of c65c1da21ae3e3e43d67b7de8f358916 into 92b2fcf2928c41bc82fd5a15d4107a21(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:11,037 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:11,037 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/C, priority=13, startTime=1733201050885; duration=0sec 2024-12-03T04:44:11,037 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:11,037 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:C 2024-12-03T04:44:11,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742065_1241 (size=12151) 2024-12-03T04:44:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:11,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:11,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201111119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T04:44:11,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201111231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:11,445 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/23aaf77f729e4746a942d1e5f233449f 2024-12-03T04:44:11,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201111442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:11,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/0df4fb73dee143828ec8f487d4c2844c is 50, key is test_row_0/B:col10/1733201049926/Put/seqid=0 2024-12-03T04:44:11,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742066_1242 (size=12151) 2024-12-03T04:44:11,468 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/0df4fb73dee143828ec8f487d4c2844c 2024-12-03T04:44:11,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/a879f505a77449059769c8aca2a12c3f is 50, key is test_row_0/C:col10/1733201049926/Put/seqid=0 2024-12-03T04:44:11,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742067_1243 (size=12151) 2024-12-03T04:44:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T04:44:11,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201111751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:11,881 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/a879f505a77449059769c8aca2a12c3f 2024-12-03T04:44:11,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/23aaf77f729e4746a942d1e5f233449f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/23aaf77f729e4746a942d1e5f233449f 2024-12-03T04:44:11,890 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/23aaf77f729e4746a942d1e5f233449f, entries=150, sequenceid=165, filesize=11.9 K 2024-12-03T04:44:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/0df4fb73dee143828ec8f487d4c2844c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0df4fb73dee143828ec8f487d4c2844c 2024-12-03T04:44:11,895 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0df4fb73dee143828ec8f487d4c2844c, entries=150, sequenceid=165, filesize=11.9 K 2024-12-03T04:44:11,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/a879f505a77449059769c8aca2a12c3f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a879f505a77449059769c8aca2a12c3f 2024-12-03T04:44:11,900 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a879f505a77449059769c8aca2a12c3f, entries=150, sequenceid=165, filesize=11.9 K 2024-12-03T04:44:11,901 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for c65c1da21ae3e3e43d67b7de8f358916 in 892ms, sequenceid=165, compaction requested=false 2024-12-03T04:44:11,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:11,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:11,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-03T04:44:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-03T04:44:11,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-03T04:44:11,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3580 sec 2024-12-03T04:44:11,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.3630 sec 2024-12-03T04:44:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:12,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:44:12,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:12,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:12,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:12,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:12,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:12,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:12,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/244b5aa2f0834e78ad73d8e8934708af is 50, key is test_row_0/A:col10/1733201051117/Put/seqid=0 2024-12-03T04:44:12,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742068_1244 (size=12151) 2024-12-03T04:44:12,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/244b5aa2f0834e78ad73d8e8934708af 2024-12-03T04:44:12,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/278df0920d674f73ad606c03a4df22c5 is 50, key is test_row_0/B:col10/1733201051117/Put/seqid=0 2024-12-03T04:44:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201112099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733201052089, max timestamp=1733201052102Num uniques 859; 2024-12-03T04:44:12,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T04:44:12,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201052095, max timestamp=1733201052102Num uniques 859; 2024-12-03T04:44:12,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201052095, max timestamp=1733201052102Num uniques 859; 2024-12-03T04:44:12,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:12,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742069_1245 (size=12151) 2024-12-03T04:44:12,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/278df0920d674f73ad606c03a4df22c5 2024-12-03T04:44:12,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/d4ac8837e5c0409b80b1c8ccca8275b4 is 50, key is test_row_0/C:col10/1733201051117/Put/seqid=0 2024-12-03T04:44:12,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201112141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201112142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201112143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742070_1246 (size=12151) 2024-12-03T04:44:12,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/d4ac8837e5c0409b80b1c8ccca8275b4 2024-12-03T04:44:12,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/244b5aa2f0834e78ad73d8e8934708af as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/244b5aa2f0834e78ad73d8e8934708af 2024-12-03T04:44:12,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/244b5aa2f0834e78ad73d8e8934708af, entries=150, sequenceid=179, filesize=11.9 K 2024-12-03T04:44:12,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/278df0920d674f73ad606c03a4df22c5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/278df0920d674f73ad606c03a4df22c5 2024-12-03T04:44:12,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/278df0920d674f73ad606c03a4df22c5, entries=150, sequenceid=179, filesize=11.9 K 2024-12-03T04:44:12,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/d4ac8837e5c0409b80b1c8ccca8275b4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d4ac8837e5c0409b80b1c8ccca8275b4 2024-12-03T04:44:12,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d4ac8837e5c0409b80b1c8ccca8275b4, entries=150, sequenceid=179, filesize=11.9 K 2024-12-03T04:44:12,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.53 KB/121370 for c65c1da21ae3e3e43d67b7de8f358916 in 122ms, sequenceid=179, compaction requested=true 2024-12-03T04:44:12,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:12,185 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:12,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:12,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:12,186 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:12,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:12,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:12,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:12,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:12,187 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:12,187 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/A is initiating minor compaction (all files) 2024-12-03T04:44:12,187 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:12,187 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/A in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:12,187 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/B is initiating minor compaction (all files) 2024-12-03T04:44:12,187 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c71c99a7b87243b29f2b594cdb86ea89, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/23aaf77f729e4746a942d1e5f233449f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/244b5aa2f0834e78ad73d8e8934708af] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=35.7 K 2024-12-03T04:44:12,187 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/B in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:12,187 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/595b7db7797642628bc5a0d187f5f0f1, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0df4fb73dee143828ec8f487d4c2844c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/278df0920d674f73ad606c03a4df22c5] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=35.7 K 2024-12-03T04:44:12,187 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting c71c99a7b87243b29f2b594cdb86ea89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733201048469 2024-12-03T04:44:12,188 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 595b7db7797642628bc5a0d187f5f0f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733201048469 2024-12-03T04:44:12,188 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23aaf77f729e4746a942d1e5f233449f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733201049893 2024-12-03T04:44:12,188 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df4fb73dee143828ec8f487d4c2844c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733201049893 2024-12-03T04:44:12,188 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 244b5aa2f0834e78ad73d8e8934708af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733201051107 2024-12-03T04:44:12,189 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 278df0920d674f73ad606c03a4df22c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733201051107 2024-12-03T04:44:12,213 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#A#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:12,214 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f60027b5e1d643dea34d867e39a27180 is 50, key is test_row_0/A:col10/1733201051117/Put/seqid=0 2024-12-03T04:44:12,215 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#B#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:12,215 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/f85b8d81598b46cf92302fd5f8cfb866 is 50, key is test_row_0/B:col10/1733201051117/Put/seqid=0 2024-12-03T04:44:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:12,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-12-03T04:44:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:12,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/185bbccbfeb64e5b8a8212b7c65afe4b is 50, key is test_row_0/A:col10/1733201052217/Put/seqid=0 2024-12-03T04:44:12,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742071_1247 (size=12459) 2024-12-03T04:44:12,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742072_1248 (size=12459) 2024-12-03T04:44:12,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201112261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201112262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201112264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201112269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201112269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742073_1249 (size=14541) 2024-12-03T04:44:12,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201112372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201112381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201112381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201112382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201112471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201112582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201112591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201112591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201112591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,646 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/f85b8d81598b46cf92302fd5f8cfb866 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f85b8d81598b46cf92302fd5f8cfb866 2024-12-03T04:44:12,651 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/B of c65c1da21ae3e3e43d67b7de8f358916 into f85b8d81598b46cf92302fd5f8cfb866(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:12,652 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:12,652 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/B, priority=13, startTime=1733201052186; duration=0sec 2024-12-03T04:44:12,652 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:12,652 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:B 2024-12-03T04:44:12,652 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:12,653 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:12,653 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/C is initiating minor compaction (all files) 2024-12-03T04:44:12,653 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/C in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:12,654 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/92b2fcf2928c41bc82fd5a15d4107a21, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a879f505a77449059769c8aca2a12c3f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d4ac8837e5c0409b80b1c8ccca8275b4] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=35.7 K 2024-12-03T04:44:12,654 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 92b2fcf2928c41bc82fd5a15d4107a21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733201048469 2024-12-03T04:44:12,655 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a879f505a77449059769c8aca2a12c3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733201049893 2024-12-03T04:44:12,656 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting d4ac8837e5c0409b80b1c8ccca8275b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733201051107 2024-12-03T04:44:12,658 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f60027b5e1d643dea34d867e39a27180 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f60027b5e1d643dea34d867e39a27180 2024-12-03T04:44:12,664 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/A of c65c1da21ae3e3e43d67b7de8f358916 into f60027b5e1d643dea34d867e39a27180(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:12,664 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:12,664 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/A, priority=13, startTime=1733201052185; duration=0sec 2024-12-03T04:44:12,664 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:12,664 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:A 2024-12-03T04:44:12,667 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#C#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:12,668 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/d88c8e01c9134bb2b525ccc191ef5518 is 50, key is test_row_0/C:col10/1733201051117/Put/seqid=0 2024-12-03T04:44:12,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742074_1250 (size=12459) 2024-12-03T04:44:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T04:44:12,704 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:12,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/185bbccbfeb64e5b8a8212b7c65afe4b 2024-12-03T04:44:12,705 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-03T04:44:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T04:44:12,708 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:12,708 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:12,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:12,734 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/d88c8e01c9134bb2b525ccc191ef5518 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d88c8e01c9134bb2b525ccc191ef5518 2024-12-03T04:44:12,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/0e5c9c7a539141ac9ad73f076e52c93a is 50, key is test_row_0/B:col10/1733201052217/Put/seqid=0 2024-12-03T04:44:12,753 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/C of c65c1da21ae3e3e43d67b7de8f358916 into d88c8e01c9134bb2b525ccc191ef5518(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:12,753 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:12,753 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/C, priority=13, startTime=1733201052186; duration=0sec 2024-12-03T04:44:12,753 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:12,753 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:C 2024-12-03T04:44:12,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742075_1251 (size=12151) 2024-12-03T04:44:12,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/0e5c9c7a539141ac9ad73f076e52c93a 2024-12-03T04:44:12,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/240bb5eb6c87422d93f4c31934d13874 is 50, key is test_row_0/C:col10/1733201052217/Put/seqid=0 2024-12-03T04:44:12,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201112782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T04:44:12,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742076_1252 (size=12151) 2024-12-03T04:44:12,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:12,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:12,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:12,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:12,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:12,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:12,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201112892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201112902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201112902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201112903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:13,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T04:44:13,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/240bb5eb6c87422d93f4c31934d13874 2024-12-03T04:44:13,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/185bbccbfeb64e5b8a8212b7c65afe4b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/185bbccbfeb64e5b8a8212b7c65afe4b 2024-12-03T04:44:13,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/185bbccbfeb64e5b8a8212b7c65afe4b, entries=200, sequenceid=214, filesize=14.2 K 2024-12-03T04:44:13,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/0e5c9c7a539141ac9ad73f076e52c93a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0e5c9c7a539141ac9ad73f076e52c93a 2024-12-03T04:44:13,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0e5c9c7a539141ac9ad73f076e52c93a, entries=150, sequenceid=214, filesize=11.9 K 2024-12-03T04:44:13,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/240bb5eb6c87422d93f4c31934d13874 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/240bb5eb6c87422d93f4c31934d13874 2024-12-03T04:44:13,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/240bb5eb6c87422d93f4c31934d13874, entries=150, sequenceid=214, filesize=11.9 K 2024-12-03T04:44:13,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for c65c1da21ae3e3e43d67b7de8f358916 in 1020ms, sequenceid=214, compaction requested=false 2024-12-03T04:44:13,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:13,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:44:13,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:13,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:13,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:13,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:13,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:13,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:13,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f24c5838c1014710becca50f98f908ff is 50, key is test_row_0/A:col10/1733201052266/Put/seqid=0 2024-12-03T04:44:13,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742077_1253 (size=12151) 2024-12-03T04:44:13,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T04:44:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201113395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201113412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201113412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201113412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:13,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201113421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:13,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:13,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:13,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201113511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:13,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:13,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:13,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f24c5838c1014710becca50f98f908ff 2024-12-03T04:44:13,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/c09fc02c13da40eb9dffe725fa19862d is 50, key is test_row_0/B:col10/1733201052266/Put/seqid=0 2024-12-03T04:44:13,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201113722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:13,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742078_1254 (size=12151) 2024-12-03T04:44:13,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/c09fc02c13da40eb9dffe725fa19862d 2024-12-03T04:44:13,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/9cb9036223dc40f0a907e7f75cac7a20 is 50, key is test_row_0/C:col10/1733201052266/Put/seqid=0 2024-12-03T04:44:13,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742079_1255 (size=12151) 2024-12-03T04:44:13,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/9cb9036223dc40f0a907e7f75cac7a20 2024-12-03T04:44:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T04:44:13,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f24c5838c1014710becca50f98f908ff as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f24c5838c1014710becca50f98f908ff 2024-12-03T04:44:13,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f24c5838c1014710becca50f98f908ff, entries=150, sequenceid=229, filesize=11.9 K 2024-12-03T04:44:13,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/c09fc02c13da40eb9dffe725fa19862d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/c09fc02c13da40eb9dffe725fa19862d 2024-12-03T04:44:13,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:13,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/c09fc02c13da40eb9dffe725fa19862d, entries=150, sequenceid=229, filesize=11.9 K 2024-12-03T04:44:13,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:13,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/9cb9036223dc40f0a907e7f75cac7a20 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/9cb9036223dc40f0a907e7f75cac7a20 2024-12-03T04:44:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:13,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/9cb9036223dc40f0a907e7f75cac7a20, entries=150, sequenceid=229, filesize=11.9 K 2024-12-03T04:44:13,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c65c1da21ae3e3e43d67b7de8f358916 in 681ms, sequenceid=229, compaction requested=true 2024-12-03T04:44:13,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:13,974 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:13,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:13,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:13,974 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:13,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:13,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:13,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:13,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:13,981 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:13,981 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/A is initiating minor compaction (all files) 2024-12-03T04:44:13,981 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/A in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,982 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f60027b5e1d643dea34d867e39a27180, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/185bbccbfeb64e5b8a8212b7c65afe4b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f24c5838c1014710becca50f98f908ff] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=38.2 K 2024-12-03T04:44:13,985 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:13,985 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f60027b5e1d643dea34d867e39a27180, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733201051107 2024-12-03T04:44:13,985 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/B is initiating minor compaction (all files) 2024-12-03T04:44:13,985 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/B in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:13,985 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f85b8d81598b46cf92302fd5f8cfb866, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0e5c9c7a539141ac9ad73f076e52c93a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/c09fc02c13da40eb9dffe725fa19862d] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=35.9 K 2024-12-03T04:44:13,989 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting f85b8d81598b46cf92302fd5f8cfb866, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733201051107 2024-12-03T04:44:13,989 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185bbccbfeb64e5b8a8212b7c65afe4b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733201052130 2024-12-03T04:44:13,997 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e5c9c7a539141ac9ad73f076e52c93a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733201052130 2024-12-03T04:44:13,997 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f24c5838c1014710becca50f98f908ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201052266 2024-12-03T04:44:14,001 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c09fc02c13da40eb9dffe725fa19862d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201052266 2024-12-03T04:44:14,058 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#B#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:14,059 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/bbc8f0aab20f4450a14bd260afd4758b is 50, key is test_row_0/B:col10/1733201052266/Put/seqid=0 2024-12-03T04:44:14,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201053358, max timestamp=1733201054052Num uniques 859; 2024-12-03T04:44:14,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:14,064 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#A#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:14,064 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/5025fbbd0ed5428ebadf0e95f0b034d4 is 50, key is test_row_0/A:col10/1733201052266/Put/seqid=0 2024-12-03T04:44:14,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201053358, max timestamp=1733201054052Num uniques 859; 2024-12-03T04:44:14,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:14,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-03T04:44:14,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:14,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:14,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:14,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:14,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:14,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:14,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/02535456923d4ddc805cff64c2d8c29b is 50, key is test_row_0/A:col10/1733201054052/Put/seqid=0 2024-12-03T04:44:14,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742080_1256 (size=12561) 2024-12-03T04:44:14,099 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/bbc8f0aab20f4450a14bd260afd4758b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/bbc8f0aab20f4450a14bd260afd4758b 2024-12-03T04:44:14,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742081_1257 (size=12561) 2024-12-03T04:44:14,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742082_1258 (size=12151) 2024-12-03T04:44:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/02535456923d4ddc805cff64c2d8c29b 2024-12-03T04:44:14,111 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/B of c65c1da21ae3e3e43d67b7de8f358916 into bbc8f0aab20f4450a14bd260afd4758b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:14,111 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:14,111 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/B, priority=13, startTime=1733201053974; duration=0sec 2024-12-03T04:44:14,111 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:14,111 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:B 2024-12-03T04:44:14,112 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:14,114 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:14,114 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/C is initiating minor compaction (all files) 2024-12-03T04:44:14,114 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/C in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,115 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d88c8e01c9134bb2b525ccc191ef5518, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/240bb5eb6c87422d93f4c31934d13874, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/9cb9036223dc40f0a907e7f75cac7a20] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=35.9 K 2024-12-03T04:44:14,116 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting d88c8e01c9134bb2b525ccc191ef5518, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733201051107 2024-12-03T04:44:14,117 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 240bb5eb6c87422d93f4c31934d13874, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733201052130 2024-12-03T04:44:14,117 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb9036223dc40f0a907e7f75cac7a20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201052266 2024-12-03T04:44:14,123 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/5025fbbd0ed5428ebadf0e95f0b034d4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/5025fbbd0ed5428ebadf0e95f0b034d4 2024-12-03T04:44:14,131 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/A of c65c1da21ae3e3e43d67b7de8f358916 into 5025fbbd0ed5428ebadf0e95f0b034d4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:14,131 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:14,131 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/A, priority=13, startTime=1733201053974; duration=0sec 2024-12-03T04:44:14,131 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:14,131 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:A 2024-12-03T04:44:14,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/f4903997a0fe4482b8a0eaa6fd3f66ac is 50, key is test_row_0/B:col10/1733201054052/Put/seqid=0 2024-12-03T04:44:14,143 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#C#compaction#217 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:14,143 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/f296d3fd5d3b4ea385285f77a88d5839 is 50, key is test_row_0/C:col10/1733201052266/Put/seqid=0 2024-12-03T04:44:14,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742083_1259 (size=12151) 2024-12-03T04:44:14,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/f4903997a0fe4482b8a0eaa6fd3f66ac 2024-12-03T04:44:14,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742084_1260 (size=12561) 2024-12-03T04:44:14,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/492aefac32e548ff9b9293d27b516c61 is 50, key is test_row_0/C:col10/1733201054052/Put/seqid=0 2024-12-03T04:44:14,207 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/f296d3fd5d3b4ea385285f77a88d5839 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/f296d3fd5d3b4ea385285f77a88d5839 2024-12-03T04:44:14,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201054176, max timestamp=1733201054207Num uniques 859; 2024-12-03T04:44:14,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:14,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201054176, max timestamp=1733201054207Num uniques 859; 2024-12-03T04:44:14,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:14,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201054176, max timestamp=1733201054207Num uniques 859; 2024-12-03T04:44:14,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:14,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742085_1261 (size=12151) 2024-12-03T04:44:14,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/492aefac32e548ff9b9293d27b516c61 2024-12-03T04:44:14,221 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/C of c65c1da21ae3e3e43d67b7de8f358916 into f296d3fd5d3b4ea385285f77a88d5839(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:14,221 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:14,222 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/C, priority=13, startTime=1733201053975; duration=0sec 2024-12-03T04:44:14,222 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:14,222 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:C 2024-12-03T04:44:14,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/02535456923d4ddc805cff64c2d8c29b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/02535456923d4ddc805cff64c2d8c29b 2024-12-03T04:44:14,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/02535456923d4ddc805cff64c2d8c29b, entries=150, sequenceid=250, filesize=11.9 K 2024-12-03T04:44:14,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/f4903997a0fe4482b8a0eaa6fd3f66ac as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f4903997a0fe4482b8a0eaa6fd3f66ac 2024-12-03T04:44:14,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f4903997a0fe4482b8a0eaa6fd3f66ac, entries=150, sequenceid=250, filesize=11.9 K 2024-12-03T04:44:14,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/492aefac32e548ff9b9293d27b516c61 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/492aefac32e548ff9b9293d27b516c61 2024-12-03T04:44:14,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:14,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:14,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/492aefac32e548ff9b9293d27b516c61, entries=150, sequenceid=250, filesize=11.9 K 2024-12-03T04:44:14,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=60.38 KB/61830 for c65c1da21ae3e3e43d67b7de8f358916 in 205ms, sequenceid=250, compaction requested=false 2024-12-03T04:44:14,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:14,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=67.09 KB heapSize=185.77 KB 2024-12-03T04:44:14,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:14,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:14,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:14,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:14,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:14,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:14,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/01c410ea5a694e57bb44efe226400982 is 50, key is test_row_0/A:col10/1733201054237/Put/seqid=0 2024-12-03T04:44:14,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742086_1262 (size=12301) 2024-12-03T04:44:14,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201114410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:14,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:14,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:14,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201114432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:14,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201114432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:14,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201114432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:14,435 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:14,435 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:14,436 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:14,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201114441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:14,444 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:14,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201114522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:14,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:14,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:14,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:14,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/01c410ea5a694e57bb44efe226400982 2024-12-03T04:44:14,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:14,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201114731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:14,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/e0fb7b299f774c00be3442aef2b146ab is 50, key is test_row_0/B:col10/1733201054237/Put/seqid=0 2024-12-03T04:44:14,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742087_1263 (size=12301) 2024-12-03T04:44:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T04:44:14,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:14,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201115041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:15,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/e0fb7b299f774c00be3442aef2b146ab 2024-12-03T04:44:15,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/32cbf6cace4f4dba9a9ef0d24910614c is 50, key is test_row_0/C:col10/1733201054237/Put/seqid=0 2024-12-03T04:44:15,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742088_1264 (size=12301) 2024-12-03T04:44:15,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:15,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:15,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:15,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:15,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:15,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:15,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201115552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:15,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/32cbf6cace4f4dba9a9ef0d24910614c 2024-12-03T04:44:15,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/01c410ea5a694e57bb44efe226400982 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/01c410ea5a694e57bb44efe226400982 2024-12-03T04:44:15,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/01c410ea5a694e57bb44efe226400982, entries=150, sequenceid=280, filesize=12.0 K 2024-12-03T04:44:15,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/e0fb7b299f774c00be3442aef2b146ab as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e0fb7b299f774c00be3442aef2b146ab 2024-12-03T04:44:15,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e0fb7b299f774c00be3442aef2b146ab, entries=150, sequenceid=280, filesize=12.0 K 2024-12-03T04:44:15,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/32cbf6cace4f4dba9a9ef0d24910614c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/32cbf6cace4f4dba9a9ef0d24910614c 2024-12-03T04:44:15,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/32cbf6cace4f4dba9a9ef0d24910614c, entries=150, sequenceid=280, filesize=12.0 K 2024-12-03T04:44:15,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~185.72 KB/190176, currentSize=114.05 KB/116790 for c65c1da21ae3e3e43d67b7de8f358916 in 1335ms, sequenceid=280, compaction requested=true 2024-12-03T04:44:15,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:15,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:15,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:15,618 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:15,618 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:15,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:15,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:15,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:15,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:15,633 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:15,633 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/A is initiating minor compaction (all files) 2024-12-03T04:44:15,633 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/A in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,633 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/5025fbbd0ed5428ebadf0e95f0b034d4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/02535456923d4ddc805cff64c2d8c29b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/01c410ea5a694e57bb44efe226400982] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=36.1 K 2024-12-03T04:44:15,633 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:15,633 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/B is initiating minor compaction (all files) 2024-12-03T04:44:15,634 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/B in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,634 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5025fbbd0ed5428ebadf0e95f0b034d4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201052266 2024-12-03T04:44:15,634 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/bbc8f0aab20f4450a14bd260afd4758b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f4903997a0fe4482b8a0eaa6fd3f66ac, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e0fb7b299f774c00be3442aef2b146ab] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=36.1 K 2024-12-03T04:44:15,634 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02535456923d4ddc805cff64c2d8c29b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733201053358 2024-12-03T04:44:15,634 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc8f0aab20f4450a14bd260afd4758b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201052266 2024-12-03T04:44:15,635 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01c410ea5a694e57bb44efe226400982, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733201054220 2024-12-03T04:44:15,635 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting f4903997a0fe4482b8a0eaa6fd3f66ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733201053358 2024-12-03T04:44:15,636 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e0fb7b299f774c00be3442aef2b146ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733201054220 2024-12-03T04:44:15,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T04:44:15,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,654 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:44:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:15,656 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:15,656 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/eda9601d13b04b2fb9249535969293aa is 50, key is test_row_0/B:col10/1733201054237/Put/seqid=0 2024-12-03T04:44:15,676 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#A#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:15,676 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/34fa64f7269644a3b2236f733801beb6 is 50, key is test_row_0/A:col10/1733201054237/Put/seqid=0 2024-12-03T04:44:15,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/6e703d07233847eb85b963de34c8bae2 is 50, key is test_row_0/A:col10/1733201054408/Put/seqid=0 2024-12-03T04:44:15,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742089_1265 (size=12813) 2024-12-03T04:44:15,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742090_1266 (size=12813) 2024-12-03T04:44:15,723 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/eda9601d13b04b2fb9249535969293aa as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/eda9601d13b04b2fb9249535969293aa 2024-12-03T04:44:15,724 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/34fa64f7269644a3b2236f733801beb6 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/34fa64f7269644a3b2236f733801beb6 2024-12-03T04:44:15,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742091_1267 (size=12301) 2024-12-03T04:44:15,739 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/B of c65c1da21ae3e3e43d67b7de8f358916 into eda9601d13b04b2fb9249535969293aa(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:15,739 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:15,739 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/B, priority=13, startTime=1733201055618; duration=0sec 2024-12-03T04:44:15,739 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:15,739 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:B 2024-12-03T04:44:15,739 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:15,742 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/6e703d07233847eb85b963de34c8bae2 2024-12-03T04:44:15,746 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/A of c65c1da21ae3e3e43d67b7de8f358916 into 34fa64f7269644a3b2236f733801beb6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:15,746 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:15,746 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/A, priority=13, startTime=1733201055618; duration=0sec 2024-12-03T04:44:15,746 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:15,746 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:15,746 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:A 2024-12-03T04:44:15,746 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/C is initiating minor compaction (all files) 2024-12-03T04:44:15,746 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/C in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:15,746 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/f296d3fd5d3b4ea385285f77a88d5839, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/492aefac32e548ff9b9293d27b516c61, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/32cbf6cace4f4dba9a9ef0d24910614c] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=36.1 K 2024-12-03T04:44:15,746 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting f296d3fd5d3b4ea385285f77a88d5839, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733201052266 2024-12-03T04:44:15,749 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 492aefac32e548ff9b9293d27b516c61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733201053358 2024-12-03T04:44:15,750 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 32cbf6cace4f4dba9a9ef0d24910614c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733201054220 2024-12-03T04:44:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/7a4023c0f0fb4c3c891665aaaf934ead is 50, key is test_row_0/B:col10/1733201054408/Put/seqid=0 2024-12-03T04:44:15,773 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#C#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:15,774 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/dd7fdf3eb3124b988c9943824b789512 is 50, key is test_row_0/C:col10/1733201054237/Put/seqid=0 2024-12-03T04:44:15,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742092_1268 (size=12301) 2024-12-03T04:44:15,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742093_1269 (size=12813) 2024-12-03T04:44:16,214 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/7a4023c0f0fb4c3c891665aaaf934ead 2024-12-03T04:44:16,254 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/dd7fdf3eb3124b988c9943824b789512 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/dd7fdf3eb3124b988c9943824b789512 2024-12-03T04:44:16,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/d7afe7b7cdc94b55b90487e04fa225de is 50, key is test_row_0/C:col10/1733201054408/Put/seqid=0 2024-12-03T04:44:16,286 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/C of c65c1da21ae3e3e43d67b7de8f358916 into dd7fdf3eb3124b988c9943824b789512(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:16,286 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:16,286 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/C, priority=13, startTime=1733201055621; duration=0sec 2024-12-03T04:44:16,287 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:16,287 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:C 2024-12-03T04:44:16,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742094_1270 (size=12301) 2024-12-03T04:44:16,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:16,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201116468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201116470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201116474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201116485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201116572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,576 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201116581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201116581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201116592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201116605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,712 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/d7afe7b7cdc94b55b90487e04fa225de 2024-12-03T04:44:16,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/6e703d07233847eb85b963de34c8bae2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/6e703d07233847eb85b963de34c8bae2 2024-12-03T04:44:16,731 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/6e703d07233847eb85b963de34c8bae2, entries=150, sequenceid=300, filesize=12.0 K 2024-12-03T04:44:16,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/7a4023c0f0fb4c3c891665aaaf934ead as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/7a4023c0f0fb4c3c891665aaaf934ead 2024-12-03T04:44:16,736 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/7a4023c0f0fb4c3c891665aaaf934ead, entries=150, sequenceid=300, filesize=12.0 K 2024-12-03T04:44:16,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/d7afe7b7cdc94b55b90487e04fa225de as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d7afe7b7cdc94b55b90487e04fa225de 2024-12-03T04:44:16,745 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d7afe7b7cdc94b55b90487e04fa225de, entries=150, sequenceid=300, filesize=12.0 K 2024-12-03T04:44:16,746 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c65c1da21ae3e3e43d67b7de8f358916 in 1092ms, sequenceid=300, compaction requested=false 2024-12-03T04:44:16,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:16,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:16,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-03T04:44:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-03T04:44:16,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-03T04:44:16,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0430 sec 2024-12-03T04:44:16,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 4.0490 sec 2024-12-03T04:44:16,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T04:44:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:16,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:16,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:16,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:16,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:16,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:16,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:16,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/9604828d466f463390a7f0bb6974fe0c is 50, key is test_row_0/A:col10/1733201056794/Put/seqid=0 2024-12-03T04:44:16,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742095_1271 (size=14741) 2024-12-03T04:44:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T04:44:16,862 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:16,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-03T04:44:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T04:44:16,864 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:16,865 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:16,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:16,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201116882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201116889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201116890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201116893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T04:44:17,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201117006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201117006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201117006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201117007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T04:44:17,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:17,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:17,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:17,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:17,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T04:44:17,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T04:44:17,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:17,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:17,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:17,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:17,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:17,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201117213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201117224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201117224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201117224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/9604828d466f463390a7f0bb6974fe0c 2024-12-03T04:44:17,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/b823cc143e0a4205b100c280bed62350 is 50, key is test_row_0/B:col10/1733201056794/Put/seqid=0 2024-12-03T04:44:17,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742096_1272 (size=12301) 2024-12-03T04:44:17,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/b823cc143e0a4205b100c280bed62350 2024-12-03T04:44:17,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/913c8aa0a7a84e739cd3f6c8b8c0dd08 is 50, key is test_row_0/C:col10/1733201056794/Put/seqid=0 2024-12-03T04:44:17,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742097_1273 (size=12301) 2024-12-03T04:44:17,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/913c8aa0a7a84e739cd3f6c8b8c0dd08 2024-12-03T04:44:17,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/9604828d466f463390a7f0bb6974fe0c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9604828d466f463390a7f0bb6974fe0c 2024-12-03T04:44:17,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9604828d466f463390a7f0bb6974fe0c, entries=200, sequenceid=317, filesize=14.4 K 2024-12-03T04:44:17,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/b823cc143e0a4205b100c280bed62350 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b823cc143e0a4205b100c280bed62350 2024-12-03T04:44:17,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T04:44:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:17,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:17,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b823cc143e0a4205b100c280bed62350, entries=150, sequenceid=317, filesize=12.0 K 2024-12-03T04:44:17,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/913c8aa0a7a84e739cd3f6c8b8c0dd08 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/913c8aa0a7a84e739cd3f6c8b8c0dd08 2024-12-03T04:44:17,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/913c8aa0a7a84e739cd3f6c8b8c0dd08, entries=150, sequenceid=317, filesize=12.0 K 2024-12-03T04:44:17,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for c65c1da21ae3e3e43d67b7de8f358916 in 585ms, sequenceid=317, compaction requested=true 2024-12-03T04:44:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T04:44:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T04:44:17,380 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:17,381 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:17,382 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:17,382 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/C is initiating minor compaction (all files) 2024-12-03T04:44:17,382 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/C in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:17,382 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/dd7fdf3eb3124b988c9943824b789512, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d7afe7b7cdc94b55b90487e04fa225de, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/913c8aa0a7a84e739cd3f6c8b8c0dd08] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=36.5 K 2024-12-03T04:44:17,383 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:17,383 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/A is initiating minor compaction (all files) 2024-12-03T04:44:17,383 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/A in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:17,383 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/34fa64f7269644a3b2236f733801beb6, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/6e703d07233847eb85b963de34c8bae2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9604828d466f463390a7f0bb6974fe0c] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=38.9 K 2024-12-03T04:44:17,383 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34fa64f7269644a3b2236f733801beb6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733201054220 2024-12-03T04:44:17,384 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting dd7fdf3eb3124b988c9943824b789512, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733201054220 2024-12-03T04:44:17,384 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e703d07233847eb85b963de34c8bae2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733201054322 2024-12-03T04:44:17,384 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting d7afe7b7cdc94b55b90487e04fa225de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733201054322 2024-12-03T04:44:17,385 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9604828d466f463390a7f0bb6974fe0c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201056465 2024-12-03T04:44:17,385 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 913c8aa0a7a84e739cd3f6c8b8c0dd08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201056473 2024-12-03T04:44:17,408 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#C#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:17,408 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/6e909990a9684c8eaf32b7eae4728805 is 50, key is test_row_0/C:col10/1733201056794/Put/seqid=0 2024-12-03T04:44:17,411 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#A#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:17,412 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/c8c9e307eb2f4c6e9cfde1e31b1f6ecb is 50, key is test_row_0/A:col10/1733201056794/Put/seqid=0 2024-12-03T04:44:17,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742098_1274 (size=12915) 2024-12-03T04:44:17,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742099_1275 (size=12915) 2024-12-03T04:44:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T04:44:17,492 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/c8c9e307eb2f4c6e9cfde1e31b1f6ecb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c8c9e307eb2f4c6e9cfde1e31b1f6ecb 2024-12-03T04:44:17,501 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/A of c65c1da21ae3e3e43d67b7de8f358916 into c8c9e307eb2f4c6e9cfde1e31b1f6ecb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:17,501 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:17,501 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/A, priority=13, startTime=1733201057380; duration=0sec 2024-12-03T04:44:17,501 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:17,501 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:A 2024-12-03T04:44:17,501 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:17,502 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/6e909990a9684c8eaf32b7eae4728805 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/6e909990a9684c8eaf32b7eae4728805 2024-12-03T04:44:17,503 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:17,504 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/B is initiating minor compaction (all files) 2024-12-03T04:44:17,504 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/B in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:17,504 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/eda9601d13b04b2fb9249535969293aa, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/7a4023c0f0fb4c3c891665aaaf934ead, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b823cc143e0a4205b100c280bed62350] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=36.5 K 2024-12-03T04:44:17,506 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting eda9601d13b04b2fb9249535969293aa, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733201054220 2024-12-03T04:44:17,507 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a4023c0f0fb4c3c891665aaaf934ead, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733201054322 2024-12-03T04:44:17,508 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting b823cc143e0a4205b100c280bed62350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201056473 2024-12-03T04:44:17,510 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/C of c65c1da21ae3e3e43d67b7de8f358916 into 6e909990a9684c8eaf32b7eae4728805(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:17,510 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:17,510 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/C, priority=13, startTime=1733201057380; duration=0sec 2024-12-03T04:44:17,510 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:17,510 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:C 2024-12-03T04:44:17,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T04:44:17,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:17,515 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T04:44:17,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:17,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:17,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:17,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:17,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:17,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:17,521 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#B#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:17,521 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/b252ab4a4c384771b5ca16f115964518 is 50, key is test_row_0/B:col10/1733201056794/Put/seqid=0 2024-12-03T04:44:17,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/4a49332a30f34fb19e99b1ef28324a50 is 50, key is test_row_0/A:col10/1733201056845/Put/seqid=0 2024-12-03T04:44:17,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:17,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742100_1276 (size=12301) 2024-12-03T04:44:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:17,534 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/4a49332a30f34fb19e99b1ef28324a50 2024-12-03T04:44:17,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/72177cf0756a4a608db3bb2d70cd7cbf is 50, key is test_row_0/B:col10/1733201056845/Put/seqid=0 2024-12-03T04:44:17,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742102_1278 (size=12301) 2024-12-03T04:44:17,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742101_1277 (size=12915) 2024-12-03T04:44:17,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201117559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201117561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201117562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201117563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201117673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201117673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201117678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201117678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201117882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201117884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201117897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201117897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:17,960 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/72177cf0756a4a608db3bb2d70cd7cbf 2024-12-03T04:44:17,966 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/b252ab4a4c384771b5ca16f115964518 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b252ab4a4c384771b5ca16f115964518 2024-12-03T04:44:17,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/85204f329fe94546bce0a87ee5748101 is 50, key is test_row_0/C:col10/1733201056845/Put/seqid=0 2024-12-03T04:44:17,974 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/B of c65c1da21ae3e3e43d67b7de8f358916 into b252ab4a4c384771b5ca16f115964518(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:17,974 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:17,974 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/B, priority=13, startTime=1733201057380; duration=0sec 2024-12-03T04:44:17,975 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:17,975 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:B 2024-12-03T04:44:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T04:44:18,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742103_1279 (size=12301) 2024-12-03T04:44:18,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201118191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:18,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:18,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201118201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:18,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:18,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201118212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:18,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201118212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:18,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:18,414 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/85204f329fe94546bce0a87ee5748101 2024-12-03T04:44:18,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/4a49332a30f34fb19e99b1ef28324a50 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/4a49332a30f34fb19e99b1ef28324a50 2024-12-03T04:44:18,444 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/4a49332a30f34fb19e99b1ef28324a50, entries=150, sequenceid=338, filesize=12.0 K 2024-12-03T04:44:18,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/72177cf0756a4a608db3bb2d70cd7cbf as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/72177cf0756a4a608db3bb2d70cd7cbf 2024-12-03T04:44:18,452 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/72177cf0756a4a608db3bb2d70cd7cbf, entries=150, sequenceid=338, filesize=12.0 K 2024-12-03T04:44:18,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/85204f329fe94546bce0a87ee5748101 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/85204f329fe94546bce0a87ee5748101 2024-12-03T04:44:18,458 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/85204f329fe94546bce0a87ee5748101, entries=150, sequenceid=338, filesize=12.0 K 2024-12-03T04:44:18,463 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for c65c1da21ae3e3e43d67b7de8f358916 in 947ms, sequenceid=338, compaction requested=false 2024-12-03T04:44:18,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:18,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:18,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-03T04:44:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-03T04:44:18,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-03T04:44:18,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6010 sec 2024-12-03T04:44:18,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.6090 sec 2024-12-03T04:44:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:18,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T04:44:18,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:18,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:18,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:18,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:18,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:18,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:18,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/0aebba5368a940818b6904fa535f1fb4 is 50, key is test_row_0/A:col10/1733201057547/Put/seqid=0 2024-12-03T04:44:18,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742104_1280 (size=14741) 2024-12-03T04:44:18,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/0aebba5368a940818b6904fa535f1fb4 2024-12-03T04:44:18,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/31f778592aac469c824e36d813408478 is 50, key is test_row_0/B:col10/1733201057547/Put/seqid=0 2024-12-03T04:44:18,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742105_1281 (size=12301) 2024-12-03T04:44:18,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201118678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:18,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201118702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:18,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201118711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:18,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201118722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:18,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201118722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:18,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201118794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T04:44:19,014 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:19,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201119014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:19,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-03T04:44:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T04:44:19,029 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:19,034 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:19,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:19,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/31f778592aac469c824e36d813408478 2024-12-03T04:44:19,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/b97365ce1a3e4ec29e26647909ba2b99 is 50, key is test_row_0/C:col10/1733201057547/Put/seqid=0 2024-12-03T04:44:19,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742106_1282 (size=12301) 2024-12-03T04:44:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T04:44:19,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T04:44:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:19,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:19,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201119321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:19,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T04:44:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T04:44:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:19,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:19,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:19,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/b97365ce1a3e4ec29e26647909ba2b99 2024-12-03T04:44:19,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T04:44:19,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:19,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:19,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:19,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:19,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:19,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/0aebba5368a940818b6904fa535f1fb4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0aebba5368a940818b6904fa535f1fb4 2024-12-03T04:44:19,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0aebba5368a940818b6904fa535f1fb4, entries=200, sequenceid=354, filesize=14.4 K 2024-12-03T04:44:19,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/31f778592aac469c824e36d813408478 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/31f778592aac469c824e36d813408478 2024-12-03T04:44:19,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/31f778592aac469c824e36d813408478, entries=150, sequenceid=354, filesize=12.0 K 2024-12-03T04:44:19,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/b97365ce1a3e4ec29e26647909ba2b99 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b97365ce1a3e4ec29e26647909ba2b99 2024-12-03T04:44:19,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b97365ce1a3e4ec29e26647909ba2b99, entries=150, sequenceid=354, filesize=12.0 K 2024-12-03T04:44:19,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c65c1da21ae3e3e43d67b7de8f358916 in 968ms, sequenceid=354, compaction requested=true 2024-12-03T04:44:19,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:19,574 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:19,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:19,575 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:19,575 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/A is initiating minor compaction (all files) 2024-12-03T04:44:19,575 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/A in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:19,576 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c8c9e307eb2f4c6e9cfde1e31b1f6ecb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/4a49332a30f34fb19e99b1ef28324a50, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0aebba5368a940818b6904fa535f1fb4] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=39.0 K 2024-12-03T04:44:19,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:19,576 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:19,576 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8c9e307eb2f4c6e9cfde1e31b1f6ecb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201056473 2024-12-03T04:44:19,577 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a49332a30f34fb19e99b1ef28324a50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733201056839 2024-12-03T04:44:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:19,577 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aebba5368a940818b6904fa535f1fb4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733201057547 2024-12-03T04:44:19,578 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:19,578 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/B is initiating minor compaction (all files) 2024-12-03T04:44:19,578 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/B in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:19,578 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b252ab4a4c384771b5ca16f115964518, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/72177cf0756a4a608db3bb2d70cd7cbf, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/31f778592aac469c824e36d813408478] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=36.6 K 2024-12-03T04:44:19,579 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting b252ab4a4c384771b5ca16f115964518, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201056473 2024-12-03T04:44:19,580 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 72177cf0756a4a608db3bb2d70cd7cbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733201056839 2024-12-03T04:44:19,581 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f778592aac469c824e36d813408478, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733201057547 2024-12-03T04:44:19,591 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#A#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:19,591 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:19,592 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d2a055ced57247a396c71757429b0de8 is 50, key is test_row_0/A:col10/1733201057547/Put/seqid=0 2024-12-03T04:44:19,592 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/e381dd6cd6254117b7675e9d4303fbac is 50, key is test_row_0/B:col10/1733201057547/Put/seqid=0 2024-12-03T04:44:19,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742107_1283 (size=13017) 2024-12-03T04:44:19,624 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d2a055ced57247a396c71757429b0de8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d2a055ced57247a396c71757429b0de8 2024-12-03T04:44:19,630 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/A of c65c1da21ae3e3e43d67b7de8f358916 into d2a055ced57247a396c71757429b0de8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:19,630 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:19,630 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/A, priority=13, startTime=1733201059574; duration=0sec 2024-12-03T04:44:19,630 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:19,630 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:A 2024-12-03T04:44:19,630 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:19,631 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:19,631 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/C is initiating minor compaction (all files) 2024-12-03T04:44:19,631 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/C in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:19,632 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/6e909990a9684c8eaf32b7eae4728805, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/85204f329fe94546bce0a87ee5748101, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b97365ce1a3e4ec29e26647909ba2b99] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=36.6 K 2024-12-03T04:44:19,632 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e909990a9684c8eaf32b7eae4728805, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733201056473 2024-12-03T04:44:19,633 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85204f329fe94546bce0a87ee5748101, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733201056839 2024-12-03T04:44:19,633 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting b97365ce1a3e4ec29e26647909ba2b99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733201057547 2024-12-03T04:44:19,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742108_1284 (size=13017) 2024-12-03T04:44:19,644 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/e381dd6cd6254117b7675e9d4303fbac as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e381dd6cd6254117b7675e9d4303fbac 2024-12-03T04:44:19,648 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:19,649 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/1fbfea4281ed4f29b3db551e673d250b is 50, key is test_row_0/C:col10/1733201057547/Put/seqid=0 2024-12-03T04:44:19,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T04:44:19,652 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/B of c65c1da21ae3e3e43d67b7de8f358916 into e381dd6cd6254117b7675e9d4303fbac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:19,652 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:19,652 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/B, priority=13, startTime=1733201059576; duration=0sec 2024-12-03T04:44:19,653 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:19,653 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:B 2024-12-03T04:44:19,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T04:44:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:19,659 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T04:44:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:19,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:19,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:19,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:19,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742109_1285 (size=13017) 2024-12-03T04:44:19,673 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/1fbfea4281ed4f29b3db551e673d250b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/1fbfea4281ed4f29b3db551e673d250b 2024-12-03T04:44:19,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/c8fd09d09a5b40efb791b811a48b39c3 is 50, key is test_row_0/A:col10/1733201058676/Put/seqid=0 2024-12-03T04:44:19,680 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/C of c65c1da21ae3e3e43d67b7de8f358916 into 1fbfea4281ed4f29b3db551e673d250b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:19,680 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:19,680 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/C, priority=13, startTime=1733201059577; duration=0sec 2024-12-03T04:44:19,680 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:19,680 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:C 2024-12-03T04:44:19,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742110_1286 (size=12301) 2024-12-03T04:44:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:19,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:19,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:19,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201119745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201119747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:19,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201119748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:19,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201119752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:19,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201119831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201119861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201119862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201119862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201119862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201120072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201120072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201120072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201120072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,101 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/c8fd09d09a5b40efb791b811a48b39c3 2024-12-03T04:44:20,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/59b96bcb3fff49bc8317c1e406550bf8 is 50, key is test_row_0/B:col10/1733201058676/Put/seqid=0 2024-12-03T04:44:20,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742111_1287 (size=12301) 2024-12-03T04:44:20,128 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/59b96bcb3fff49bc8317c1e406550bf8 2024-12-03T04:44:20,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/69cc1ba64f1d4bc38d6c25fad798fde7 is 50, key is test_row_0/C:col10/1733201058676/Put/seqid=0 2024-12-03T04:44:20,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742112_1288 (size=12301) 2024-12-03T04:44:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T04:44:20,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201120381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201120381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201120396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201120396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,547 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/69cc1ba64f1d4bc38d6c25fad798fde7 2024-12-03T04:44:20,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/c8fd09d09a5b40efb791b811a48b39c3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c8fd09d09a5b40efb791b811a48b39c3 2024-12-03T04:44:20,564 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c8fd09d09a5b40efb791b811a48b39c3, entries=150, sequenceid=373, filesize=12.0 K 2024-12-03T04:44:20,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/59b96bcb3fff49bc8317c1e406550bf8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/59b96bcb3fff49bc8317c1e406550bf8 2024-12-03T04:44:20,577 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/59b96bcb3fff49bc8317c1e406550bf8, entries=150, sequenceid=373, filesize=12.0 K 2024-12-03T04:44:20,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/69cc1ba64f1d4bc38d6c25fad798fde7 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/69cc1ba64f1d4bc38d6c25fad798fde7 2024-12-03T04:44:20,593 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/69cc1ba64f1d4bc38d6c25fad798fde7, entries=150, sequenceid=373, filesize=12.0 K 2024-12-03T04:44:20,594 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c65c1da21ae3e3e43d67b7de8f358916 in 935ms, sequenceid=373, compaction requested=false 2024-12-03T04:44:20,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:20,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:20,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-03T04:44:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-03T04:44:20,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-03T04:44:20,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5610 sec 2024-12-03T04:44:20,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5780 sec 2024-12-03T04:44:20,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T04:44:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:20,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:20,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/9c4a0a74696b45b6b6fcb4345c1ef75a is 50, key is test_row_0/A:col10/1733201059751/Put/seqid=0 2024-12-03T04:44:20,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742113_1289 (size=12301) 2024-12-03T04:44:20,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201120910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201120911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201120912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201120912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201120913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201121021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201121021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201121021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201121021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T04:44:21,172 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:21,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-03T04:44:21,175 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T04:44:21,176 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:21,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:21,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201121232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201121232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201121232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201121232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T04:44:21,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/9c4a0a74696b45b6b6fcb4345c1ef75a 2024-12-03T04:44:21,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/ebd8924d37c24bdeadd5bb30d6bbeefc is 50, key is test_row_0/B:col10/1733201059751/Put/seqid=0 2024-12-03T04:44:21,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742114_1290 (size=12301) 2024-12-03T04:44:21,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T04:44:21,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:21,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:21,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:21,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T04:44:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:21,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T04:44:21,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201121541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201121542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201121542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201121542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T04:44:21,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:21,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:21,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:21,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/ebd8924d37c24bdeadd5bb30d6bbeefc 2024-12-03T04:44:21,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/78e6375d1de34c05b1e1f4e20d16aaaa is 50, key is test_row_0/C:col10/1733201059751/Put/seqid=0 2024-12-03T04:44:21,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742115_1291 (size=12301) 2024-12-03T04:44:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T04:44:21,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T04:44:21,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:21,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:21,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:21,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201121932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:21,937 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:21,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T04:44:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:21,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:22,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201122051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:22,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201122051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:22,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201122061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:22,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:22,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201122062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:22,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:22,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:22,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:22,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:22,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:22,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T04:44:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:22,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:22,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/78e6375d1de34c05b1e1f4e20d16aaaa 2024-12-03T04:44:22,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/9c4a0a74696b45b6b6fcb4345c1ef75a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9c4a0a74696b45b6b6fcb4345c1ef75a 2024-12-03T04:44:22,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9c4a0a74696b45b6b6fcb4345c1ef75a, entries=150, sequenceid=390, filesize=12.0 K 2024-12-03T04:44:22,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/ebd8924d37c24bdeadd5bb30d6bbeefc as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ebd8924d37c24bdeadd5bb30d6bbeefc 2024-12-03T04:44:22,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ebd8924d37c24bdeadd5bb30d6bbeefc, entries=150, sequenceid=390, filesize=12.0 K 2024-12-03T04:44:22,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/78e6375d1de34c05b1e1f4e20d16aaaa as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/78e6375d1de34c05b1e1f4e20d16aaaa 2024-12-03T04:44:22,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/78e6375d1de34c05b1e1f4e20d16aaaa, entries=150, sequenceid=390, filesize=12.0 K 2024-12-03T04:44:22,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for c65c1da21ae3e3e43d67b7de8f358916 in 1349ms, sequenceid=390, compaction requested=true 2024-12-03T04:44:22,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:22,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:22,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:22,205 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:22,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:22,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:22,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:22,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:22,205 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:22,206 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:22,206 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:22,206 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/A is initiating minor compaction (all files) 2024-12-03T04:44:22,206 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/B is initiating minor compaction (all files) 2024-12-03T04:44:22,206 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/A in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:22,206 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/B in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:22,206 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d2a055ced57247a396c71757429b0de8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c8fd09d09a5b40efb791b811a48b39c3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9c4a0a74696b45b6b6fcb4345c1ef75a] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=36.7 K 2024-12-03T04:44:22,206 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e381dd6cd6254117b7675e9d4303fbac, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/59b96bcb3fff49bc8317c1e406550bf8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ebd8924d37c24bdeadd5bb30d6bbeefc] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=36.7 K 2024-12-03T04:44:22,210 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a055ced57247a396c71757429b0de8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733201057547 2024-12-03T04:44:22,210 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e381dd6cd6254117b7675e9d4303fbac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733201057547 2024-12-03T04:44:22,211 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8fd09d09a5b40efb791b811a48b39c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733201058638 2024-12-03T04:44:22,211 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 59b96bcb3fff49bc8317c1e406550bf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733201058638 2024-12-03T04:44:22,211 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ebd8924d37c24bdeadd5bb30d6bbeefc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733201059743 2024-12-03T04:44:22,212 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c4a0a74696b45b6b6fcb4345c1ef75a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733201059743 2024-12-03T04:44:22,235 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:22,236 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/da28bafa0e8743b2a28fa4eea22ac116 is 50, key is test_row_0/A:col10/1733201059751/Put/seqid=0 2024-12-03T04:44:22,243 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:22,244 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/ec5d6e6bc67949b7b31e46c86c424098 is 50, key is test_row_0/B:col10/1733201059751/Put/seqid=0 2024-12-03T04:44:22,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T04:44:22,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742116_1292 (size=13119) 2024-12-03T04:44:22,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:22,289 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T04:44:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:22,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T04:44:22,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742117_1293 (size=13119) 2024-12-03T04:44:22,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d7d22482549341e0b2de5b8d56c7616d is 50, key is test_row_0/A:col10/1733201060910/Put/seqid=0 2024-12-03T04:44:22,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742118_1294 (size=12301) 2024-12-03T04:44:22,332 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d7d22482549341e0b2de5b8d56c7616d 2024-12-03T04:44:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/ce1d04dc75b64831b74fb089f494edcc is 50, key is test_row_0/B:col10/1733201060910/Put/seqid=0 2024-12-03T04:44:22,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742119_1295 (size=12301) 2024-12-03T04:44:22,344 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/ce1d04dc75b64831b74fb089f494edcc 2024-12-03T04:44:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/a9e6b539513349c6a8dd1b9d9ff85748 is 50, key is test_row_0/C:col10/1733201060910/Put/seqid=0 2024-12-03T04:44:22,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742120_1296 (size=12301) 2024-12-03T04:44:22,358 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/a9e6b539513349c6a8dd1b9d9ff85748 2024-12-03T04:44:22,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d7d22482549341e0b2de5b8d56c7616d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d7d22482549341e0b2de5b8d56c7616d 2024-12-03T04:44:22,368 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d7d22482549341e0b2de5b8d56c7616d, entries=150, sequenceid=407, filesize=12.0 K 2024-12-03T04:44:22,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/ce1d04dc75b64831b74fb089f494edcc as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ce1d04dc75b64831b74fb089f494edcc 2024-12-03T04:44:22,374 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ce1d04dc75b64831b74fb089f494edcc, entries=150, sequenceid=407, filesize=12.0 K 2024-12-03T04:44:22,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/a9e6b539513349c6a8dd1b9d9ff85748 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a9e6b539513349c6a8dd1b9d9ff85748 2024-12-03T04:44:22,378 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a9e6b539513349c6a8dd1b9d9ff85748, entries=150, sequenceid=407, filesize=12.0 K 2024-12-03T04:44:22,385 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for c65c1da21ae3e3e43d67b7de8f358916 in 96ms, sequenceid=407, compaction requested=true 2024-12-03T04:44:22,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:22,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:22,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-03T04:44:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-03T04:44:22,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-03T04:44:22,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2190 sec 2024-12-03T04:44:22,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2280 sec 2024-12-03T04:44:22,715 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/da28bafa0e8743b2a28fa4eea22ac116 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/da28bafa0e8743b2a28fa4eea22ac116 2024-12-03T04:44:22,722 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/ec5d6e6bc67949b7b31e46c86c424098 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ec5d6e6bc67949b7b31e46c86c424098 2024-12-03T04:44:22,724 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/A of c65c1da21ae3e3e43d67b7de8f358916 into da28bafa0e8743b2a28fa4eea22ac116(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:22,724 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:22,724 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/A, priority=13, startTime=1733201062205; duration=0sec 2024-12-03T04:44:22,724 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:22,724 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:A 2024-12-03T04:44:22,724 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:44:22,729 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:44:22,729 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/C is initiating minor compaction (all files) 2024-12-03T04:44:22,729 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/C in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:22,729 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/1fbfea4281ed4f29b3db551e673d250b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/69cc1ba64f1d4bc38d6c25fad798fde7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/78e6375d1de34c05b1e1f4e20d16aaaa, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a9e6b539513349c6a8dd1b9d9ff85748] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=48.8 K 2024-12-03T04:44:22,730 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fbfea4281ed4f29b3db551e673d250b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733201057547 2024-12-03T04:44:22,732 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69cc1ba64f1d4bc38d6c25fad798fde7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733201058638 2024-12-03T04:44:22,732 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78e6375d1de34c05b1e1f4e20d16aaaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733201059743 2024-12-03T04:44:22,733 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/B of c65c1da21ae3e3e43d67b7de8f358916 into ec5d6e6bc67949b7b31e46c86c424098(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:22,734 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:22,734 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/B, priority=13, startTime=1733201062205; duration=0sec 2024-12-03T04:44:22,734 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:22,734 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:B 2024-12-03T04:44:22,734 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9e6b539513349c6a8dd1b9d9ff85748, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733201060908 2024-12-03T04:44:22,759 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#C#compaction#254 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:22,760 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/d48c3a851db84f9496cf215295ac3157 is 50, key is test_row_0/C:col10/1733201060910/Put/seqid=0 2024-12-03T04:44:22,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742121_1297 (size=13153) 2024-12-03T04:44:22,812 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/d48c3a851db84f9496cf215295ac3157 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d48c3a851db84f9496cf215295ac3157 2024-12-03T04:44:22,819 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/C of c65c1da21ae3e3e43d67b7de8f358916 into d48c3a851db84f9496cf215295ac3157(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:22,819 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:22,819 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/C, priority=12, startTime=1733201062205; duration=0sec 2024-12-03T04:44:22,819 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:22,819 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:C 2024-12-03T04:44:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:23,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:44:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:23,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:23,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:23,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:23,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:23,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:23,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/5f3e8180a2294c57836fa43c8257ac65 is 50, key is test_row_0/A:col10/1733201063078/Put/seqid=0 2024-12-03T04:44:23,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742122_1298 (size=14741) 2024-12-03T04:44:23,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201123107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201063101, max timestamp=1733201063109Num uniques 859; 2024-12-03T04:44:23,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:23,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201063101, max timestamp=1733201063109Num uniques 859; 2024-12-03T04:44:23,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201063101, max timestamp=1733201063109Num uniques 859; 2024-12-03T04:44:23,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:23,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:23,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/5f3e8180a2294c57836fa43c8257ac65 2024-12-03T04:44:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/8f74deae3900461aabd9c1825d3c84cf is 50, key is test_row_0/B:col10/1733201063078/Put/seqid=0 2024-12-03T04:44:23,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742123_1299 (size=12301) 2024-12-03T04:44:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201123146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201123147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201123153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201123222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201123261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201123261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201123261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T04:44:23,322 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:23,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-03T04:44:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T04:44:23,325 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:23,326 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:23,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T04:44:23,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201123432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201123471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201123472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201123472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T04:44:23,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:23,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:23,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:23,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/8f74deae3900461aabd9c1825d3c84cf 2024-12-03T04:44:23,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/90904f872c2b4bff9b37ef53e1b5a3d5 is 50, key is test_row_0/C:col10/1733201063078/Put/seqid=0 2024-12-03T04:44:23,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742124_1300 (size=12301) 2024-12-03T04:44:23,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T04:44:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:23,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T04:44:23,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201123741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T04:44:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:23,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201123781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201123782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201123782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T04:44:23,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:23,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:23,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:23,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T04:44:23,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/90904f872c2b4bff9b37ef53e1b5a3d5 2024-12-03T04:44:23,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/5f3e8180a2294c57836fa43c8257ac65 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/5f3e8180a2294c57836fa43c8257ac65 2024-12-03T04:44:23,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42364 deadline: 1733201123962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:23,966 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:23,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/5f3e8180a2294c57836fa43c8257ac65, entries=200, sequenceid=421, filesize=14.4 K 2024-12-03T04:44:23,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/8f74deae3900461aabd9c1825d3c84cf as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/8f74deae3900461aabd9c1825d3c84cf 2024-12-03T04:44:23,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/8f74deae3900461aabd9c1825d3c84cf, entries=150, sequenceid=421, filesize=12.0 K 2024-12-03T04:44:23,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/90904f872c2b4bff9b37ef53e1b5a3d5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/90904f872c2b4bff9b37ef53e1b5a3d5 2024-12-03T04:44:23,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/90904f872c2b4bff9b37ef53e1b5a3d5, entries=150, sequenceid=421, filesize=12.0 K 2024-12-03T04:44:23,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c65c1da21ae3e3e43d67b7de8f358916 in 900ms, sequenceid=421, compaction requested=true 2024-12-03T04:44:23,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:23,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:23,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:23,979 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:23,979 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:23,980 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:23,980 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:23,980 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/A is initiating minor compaction (all files) 2024-12-03T04:44:23,980 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/B is initiating minor compaction (all files) 2024-12-03T04:44:23,980 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/A in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:23,980 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/B in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:23,980 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/da28bafa0e8743b2a28fa4eea22ac116, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d7d22482549341e0b2de5b8d56c7616d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/5f3e8180a2294c57836fa43c8257ac65] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=39.2 K 2024-12-03T04:44:23,980 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ec5d6e6bc67949b7b31e46c86c424098, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ce1d04dc75b64831b74fb089f494edcc, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/8f74deae3900461aabd9c1825d3c84cf] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=36.8 K 2024-12-03T04:44:23,980 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5d6e6bc67949b7b31e46c86c424098, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733201059743 2024-12-03T04:44:23,981 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ce1d04dc75b64831b74fb089f494edcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733201060908 2024-12-03T04:44:23,981 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f74deae3900461aabd9c1825d3c84cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733201063076 2024-12-03T04:44:23,981 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting da28bafa0e8743b2a28fa4eea22ac116, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733201059743 2024-12-03T04:44:23,982 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7d22482549341e0b2de5b8d56c7616d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733201060908 2024-12-03T04:44:23,982 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f3e8180a2294c57836fa43c8257ac65, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733201063073 2024-12-03T04:44:23,993 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:23,993 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:23,994 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/66db99f30c894d179bed5c1a9a25b89b is 50, key is test_row_0/B:col10/1733201063078/Put/seqid=0 2024-12-03T04:44:23,994 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/0a12626079674aaf9df48c8305f390ac is 50, key is test_row_0/A:col10/1733201063078/Put/seqid=0 2024-12-03T04:44:24,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742126_1302 (size=13221) 2024-12-03T04:44:24,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742125_1301 (size=13221) 2024-12-03T04:44:24,028 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/0a12626079674aaf9df48c8305f390ac as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0a12626079674aaf9df48c8305f390ac 2024-12-03T04:44:24,033 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/A of c65c1da21ae3e3e43d67b7de8f358916 into 0a12626079674aaf9df48c8305f390ac(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:24,033 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:24,033 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/A, priority=13, startTime=1733201063978; duration=0sec 2024-12-03T04:44:24,034 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:24,034 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:A 2024-12-03T04:44:24,034 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T04:44:24,034 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/66db99f30c894d179bed5c1a9a25b89b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/66db99f30c894d179bed5c1a9a25b89b 2024-12-03T04:44:24,035 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T04:44:24,035 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T04:44:24,035 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. because compaction request was cancelled 2024-12-03T04:44:24,035 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:C 2024-12-03T04:44:24,040 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/B of c65c1da21ae3e3e43d67b7de8f358916 into 66db99f30c894d179bed5c1a9a25b89b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:24,040 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:24,040 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/B, priority=13, startTime=1733201063978; duration=0sec 2024-12-03T04:44:24,041 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:24,041 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:B 2024-12-03T04:44:24,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T04:44:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:24,097 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T04:44:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f79d9fe42e4047daafe00063ae693ee9 is 50, key is test_row_0/A:col10/1733201063151/Put/seqid=0 2024-12-03T04:44:24,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742127_1303 (size=12301) 2024-12-03T04:44:24,108 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f79d9fe42e4047daafe00063ae693ee9 2024-12-03T04:44:24,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/5ad87b5d50e54a41870035346de87e03 is 50, key is test_row_0/B:col10/1733201063151/Put/seqid=0 2024-12-03T04:44:24,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742128_1304 (size=12301) 2024-12-03T04:44:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:24,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201124292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201124292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201124292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201124292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:24,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201124401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:24,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T04:44:24,521 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/5ad87b5d50e54a41870035346de87e03 2024-12-03T04:44:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/c330d8a0019d468da29b79127d338728 is 50, key is test_row_0/C:col10/1733201063151/Put/seqid=0 2024-12-03T04:44:24,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742129_1305 (size=12301) 2024-12-03T04:44:24,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201124612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:24,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201124921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:24,943 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/c330d8a0019d468da29b79127d338728 2024-12-03T04:44:24,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f79d9fe42e4047daafe00063ae693ee9 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f79d9fe42e4047daafe00063ae693ee9 2024-12-03T04:44:24,956 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f79d9fe42e4047daafe00063ae693ee9, entries=150, sequenceid=459, filesize=12.0 K 2024-12-03T04:44:24,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/5ad87b5d50e54a41870035346de87e03 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/5ad87b5d50e54a41870035346de87e03 2024-12-03T04:44:24,962 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/5ad87b5d50e54a41870035346de87e03, entries=150, sequenceid=459, filesize=12.0 K 2024-12-03T04:44:24,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/c330d8a0019d468da29b79127d338728 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/c330d8a0019d468da29b79127d338728 2024-12-03T04:44:24,969 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/c330d8a0019d468da29b79127d338728, entries=150, sequenceid=459, filesize=12.0 K 2024-12-03T04:44:24,974 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for c65c1da21ae3e3e43d67b7de8f358916 in 877ms, sequenceid=459, compaction requested=true 2024-12-03T04:44:24,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:24,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:24,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-03T04:44:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-03T04:44:24,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-03T04:44:24,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6490 sec 2024-12-03T04:44:24,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6550 sec 2024-12-03T04:44:25,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:25,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:44:25,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:25,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:25,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:25,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:25,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:25,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:25,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/aebff568d4bf4290a87eedd60a848880 is 50, key is test_row_0/A:col10/1733201065303/Put/seqid=0 2024-12-03T04:44:25,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742130_1306 (size=14741) 2024-12-03T04:44:25,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/aebff568d4bf4290a87eedd60a848880 2024-12-03T04:44:25,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/bda04b18674c42b0848e07fdeed70798 is 50, key is test_row_0/B:col10/1733201065303/Put/seqid=0 2024-12-03T04:44:25,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742131_1307 (size=12301) 2024-12-03T04:44:25,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201125336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:25,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201125338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:25,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201125340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:25,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201125431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:25,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201125458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:25,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201125465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:25,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201125466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T04:44:25,472 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:25,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-03T04:44:25,475 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T04:44:25,476 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:25,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T04:44:25,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T04:44:25,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:25,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:25,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:25,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:25,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:25,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201125672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201125672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:25,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201125673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:25,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/bda04b18674c42b0848e07fdeed70798 2024-12-03T04:44:25,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T04:44:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:25,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:25,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/1afaa5d1452949a1a293ee085b6f3905 is 50, key is test_row_0/C:col10/1733201065303/Put/seqid=0 2024-12-03T04:44:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T04:44:25,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742132_1308 (size=12301) 2024-12-03T04:44:25,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/1afaa5d1452949a1a293ee085b6f3905 2024-12-03T04:44:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/aebff568d4bf4290a87eedd60a848880 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/aebff568d4bf4290a87eedd60a848880 2024-12-03T04:44:25,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/aebff568d4bf4290a87eedd60a848880, entries=200, sequenceid=472, filesize=14.4 K 2024-12-03T04:44:25,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/bda04b18674c42b0848e07fdeed70798 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/bda04b18674c42b0848e07fdeed70798 2024-12-03T04:44:25,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/bda04b18674c42b0848e07fdeed70798, entries=150, sequenceid=472, filesize=12.0 K 2024-12-03T04:44:25,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/1afaa5d1452949a1a293ee085b6f3905 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/1afaa5d1452949a1a293ee085b6f3905 2024-12-03T04:44:25,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/1afaa5d1452949a1a293ee085b6f3905, entries=150, sequenceid=472, filesize=12.0 K 2024-12-03T04:44:25,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c65c1da21ae3e3e43d67b7de8f358916 in 524ms, sequenceid=472, compaction requested=true 2024-12-03T04:44:25,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:25,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:25,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:25,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:25,829 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:25,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:25,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:25,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:25,829 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:25,830 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:25,830 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:25,830 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/A is initiating minor compaction (all files) 2024-12-03T04:44:25,830 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/B is initiating minor compaction (all files) 2024-12-03T04:44:25,830 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/A in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:25,830 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/B in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:25,830 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0a12626079674aaf9df48c8305f390ac, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f79d9fe42e4047daafe00063ae693ee9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/aebff568d4bf4290a87eedd60a848880] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=39.3 K 2024-12-03T04:44:25,830 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/66db99f30c894d179bed5c1a9a25b89b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/5ad87b5d50e54a41870035346de87e03, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/bda04b18674c42b0848e07fdeed70798] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=36.9 K 2024-12-03T04:44:25,831 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 66db99f30c894d179bed5c1a9a25b89b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733201063076 2024-12-03T04:44:25,831 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a12626079674aaf9df48c8305f390ac, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733201063076 2024-12-03T04:44:25,832 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ad87b5d50e54a41870035346de87e03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733201063138 2024-12-03T04:44:25,832 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f79d9fe42e4047daafe00063ae693ee9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733201063138 2024-12-03T04:44:25,832 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting bda04b18674c42b0848e07fdeed70798, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733201064267 2024-12-03T04:44:25,832 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting aebff568d4bf4290a87eedd60a848880, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733201064267 2024-12-03T04:44:25,841 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#B#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:25,842 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/f6861e1e51a14977b8b5bc6a16ad9033 is 50, key is test_row_0/B:col10/1733201065303/Put/seqid=0 2024-12-03T04:44:25,845 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#A#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:25,846 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d51a0e7d5b314876b76006900ac8232a is 50, key is test_row_0/A:col10/1733201065303/Put/seqid=0 2024-12-03T04:44:25,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742133_1309 (size=13323) 2024-12-03T04:44:25,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742134_1310 (size=13323) 2024-12-03T04:44:25,861 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/f6861e1e51a14977b8b5bc6a16ad9033 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f6861e1e51a14977b8b5bc6a16ad9033 2024-12-03T04:44:25,865 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/B of c65c1da21ae3e3e43d67b7de8f358916 into f6861e1e51a14977b8b5bc6a16ad9033(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:25,865 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:25,865 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/B, priority=13, startTime=1733201065829; duration=0sec 2024-12-03T04:44:25,866 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:25,866 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:B 2024-12-03T04:44:25,866 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:44:25,868 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:44:25,868 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/C is initiating minor compaction (all files) 2024-12-03T04:44:25,868 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/C in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:25,868 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d48c3a851db84f9496cf215295ac3157, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/90904f872c2b4bff9b37ef53e1b5a3d5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/c330d8a0019d468da29b79127d338728, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/1afaa5d1452949a1a293ee085b6f3905] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=48.9 K 2024-12-03T04:44:25,868 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting d48c3a851db84f9496cf215295ac3157, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733201060908 2024-12-03T04:44:25,869 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 90904f872c2b4bff9b37ef53e1b5a3d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733201063076 2024-12-03T04:44:25,870 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c330d8a0019d468da29b79127d338728, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733201063138 2024-12-03T04:44:25,870 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 1afaa5d1452949a1a293ee085b6f3905, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733201064267 2024-12-03T04:44:25,887 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#C#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:25,888 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/ee03bb50d9d84c4eab39881a29c6150e is 50, key is test_row_0/C:col10/1733201065303/Put/seqid=0 2024-12-03T04:44:25,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742135_1311 (size=13289) 2024-12-03T04:44:25,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T04:44:25,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:25,953 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T04:44:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/281681c63e4d4c1aa685189f56257bff is 50, key is test_row_0/A:col10/1733201065339/Put/seqid=0 2024-12-03T04:44:25,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742136_1312 (size=12301) 2024-12-03T04:44:25,968 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/281681c63e4d4c1aa685189f56257bff 2024-12-03T04:44:25,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/4203c575c483418881e957e3c11a4004 is 50, key is test_row_0/B:col10/1733201065339/Put/seqid=0 2024-12-03T04:44:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:25,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. as already flushing 2024-12-03T04:44:25,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742137_1313 (size=12301) 2024-12-03T04:44:26,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201126000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201126003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201126004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T04:44:26,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201126112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201126112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201126113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,270 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/d51a0e7d5b314876b76006900ac8232a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d51a0e7d5b314876b76006900ac8232a 2024-12-03T04:44:26,276 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/A of c65c1da21ae3e3e43d67b7de8f358916 into d51a0e7d5b314876b76006900ac8232a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:26,276 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:26,276 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/A, priority=13, startTime=1733201065828; duration=0sec 2024-12-03T04:44:26,276 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:26,276 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:A 2024-12-03T04:44:26,316 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/ee03bb50d9d84c4eab39881a29c6150e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/ee03bb50d9d84c4eab39881a29c6150e 2024-12-03T04:44:26,321 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/C of c65c1da21ae3e3e43d67b7de8f358916 into ee03bb50d9d84c4eab39881a29c6150e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:26,321 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:26,321 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/C, priority=12, startTime=1733201065829; duration=0sec 2024-12-03T04:44:26,321 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:26,321 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:C 2024-12-03T04:44:26,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201126322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201126322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201126323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,388 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/4203c575c483418881e957e3c11a4004 2024-12-03T04:44:26,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/f9aef64138184e55a2bbbbb9476466cd is 50, key is test_row_0/C:col10/1733201065339/Put/seqid=0 2024-12-03T04:44:26,397 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-12-03T04:44:26,397 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:26,397 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:26,398 INFO [Thread-1050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1050. 2024-12-03T04:44:26,398 INFO [Thread-1044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1044. 2024-12-03T04:44:26,398 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:26,398 DEBUG [Thread-1050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:26,398 DEBUG [Thread-1044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:26,398 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:26,398 DEBUG [Thread-1044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:26,398 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:26,398 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:26,398 DEBUG [Thread-1044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:26,398 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:26,398 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:26,400 INFO [Thread-1048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1048. 2024-12-03T04:44:26,400 DEBUG [Thread-1048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:26,400 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:26,400 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:26,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:26,401 INFO [Thread-1046 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1046. 2024-12-03T04:44:26,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742138_1314 (size=12301) 2024-12-03T04:44:26,401 DEBUG [Thread-1046 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:26,401 DEBUG [Thread-1046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:26,401 DEBUG [Thread-1046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:26,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:26,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42330 deadline: 1733201126451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,452 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in c65c1da21ae3e3e43d67b7de8f358916 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T04:44:26,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42394 deadline: 1733201126632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42340 deadline: 1733201126632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42404 deadline: 1733201126632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:44:26,802 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/f9aef64138184e55a2bbbbb9476466cd 2024-12-03T04:44:26,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/281681c63e4d4c1aa685189f56257bff as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/281681c63e4d4c1aa685189f56257bff 2024-12-03T04:44:26,810 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/281681c63e4d4c1aa685189f56257bff, entries=150, sequenceid=492, filesize=12.0 K 2024-12-03T04:44:26,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/4203c575c483418881e957e3c11a4004 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/4203c575c483418881e957e3c11a4004 2024-12-03T04:44:26,813 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/4203c575c483418881e957e3c11a4004, entries=150, sequenceid=492, filesize=12.0 K 2024-12-03T04:44:26,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/f9aef64138184e55a2bbbbb9476466cd as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/f9aef64138184e55a2bbbbb9476466cd 2024-12-03T04:44:26,817 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/f9aef64138184e55a2bbbbb9476466cd, entries=150, sequenceid=492, filesize=12.0 K 2024-12-03T04:44:26,818 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c65c1da21ae3e3e43d67b7de8f358916 in 865ms, sequenceid=492, compaction requested=false 2024-12-03T04:44:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-03T04:44:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-03T04:44:26,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-03T04:44:26,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3430 sec 2024-12-03T04:44:26,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.3480 sec 2024-12-03T04:44:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:27,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T04:44:27,144 INFO [Thread-1041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1041. 2024-12-03T04:44:27,144 INFO [Thread-1039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1039. 2024-12-03T04:44:27,144 INFO [Thread-1035 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1035. 2024-12-03T04:44:27,144 DEBUG [Thread-1041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:27,144 DEBUG [Thread-1039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:27,144 DEBUG [Thread-1035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:27,144 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:27,144 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:27,144 DEBUG [Thread-1035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:27,144 DEBUG [Thread-1035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:27,144 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:27,144 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:27,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:27,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:27,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:27,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:27,144 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:27,144 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:27,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:27,144 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:27,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:27,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/0375c4b824874bcbbed8db339aaa1754 is 50, key is test_row_0/A:col10/1733201067142/Put/seqid=0 2024-12-03T04:44:27,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742139_1315 (size=12301) 2024-12-03T04:44:27,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/0375c4b824874bcbbed8db339aaa1754 2024-12-03T04:44:27,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/0464a2b21acb47289c7c51800d2b4b2b is 50, key is test_row_0/B:col10/1733201067142/Put/seqid=0 2024-12-03T04:44:27,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742140_1316 (size=12301) 2024-12-03T04:44:27,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/0464a2b21acb47289c7c51800d2b4b2b 2024-12-03T04:44:27,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/b0f2c7fcacb34d1bbcee0a2ac3de1c93 is 50, key is test_row_0/C:col10/1733201067142/Put/seqid=0 2024-12-03T04:44:27,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742141_1317 (size=12301) 2024-12-03T04:44:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T04:44:27,642 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:27,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/b0f2c7fcacb34d1bbcee0a2ac3de1c93 2024-12-03T04:44:27,994 INFO [Thread-1033 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1033. 2024-12-03T04:44:27,994 DEBUG [Thread-1033 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:27,994 DEBUG [Thread-1033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:27,994 DEBUG [Thread-1033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:27,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/0375c4b824874bcbbed8db339aaa1754 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0375c4b824874bcbbed8db339aaa1754 2024-12-03T04:44:28,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0375c4b824874bcbbed8db339aaa1754, entries=150, sequenceid=510, filesize=12.0 K 2024-12-03T04:44:28,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/0464a2b21acb47289c7c51800d2b4b2b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0464a2b21acb47289c7c51800d2b4b2b 2024-12-03T04:44:28,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0464a2b21acb47289c7c51800d2b4b2b, entries=150, sequenceid=510, filesize=12.0 K 2024-12-03T04:44:28,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/b0f2c7fcacb34d1bbcee0a2ac3de1c93 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b0f2c7fcacb34d1bbcee0a2ac3de1c93 2024-12-03T04:44:28,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b0f2c7fcacb34d1bbcee0a2ac3de1c93, entries=150, sequenceid=510, filesize=12.0 K 2024-12-03T04:44:28,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=6.71 KB/6870 for c65c1da21ae3e3e43d67b7de8f358916 in 871ms, sequenceid=510, compaction requested=true 2024-12-03T04:44:28,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:28,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:28,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:28,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:28,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:28,014 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:28,014 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:28,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c65c1da21ae3e3e43d67b7de8f358916:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:28,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:28,015 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:28,015 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:28,015 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/A is initiating minor compaction (all files) 2024-12-03T04:44:28,015 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/B is initiating minor compaction (all files) 2024-12-03T04:44:28,015 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/A in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:28,015 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/B in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:28,015 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d51a0e7d5b314876b76006900ac8232a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/281681c63e4d4c1aa685189f56257bff, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0375c4b824874bcbbed8db339aaa1754] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=37.0 K 2024-12-03T04:44:28,015 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f6861e1e51a14977b8b5bc6a16ad9033, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/4203c575c483418881e957e3c11a4004, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0464a2b21acb47289c7c51800d2b4b2b] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=37.0 K 2024-12-03T04:44:28,016 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting d51a0e7d5b314876b76006900ac8232a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733201064267 2024-12-03T04:44:28,016 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting f6861e1e51a14977b8b5bc6a16ad9033, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733201064267 2024-12-03T04:44:28,016 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 281681c63e4d4c1aa685189f56257bff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1733201065334 2024-12-03T04:44:28,016 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 4203c575c483418881e957e3c11a4004, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1733201065334 2024-12-03T04:44:28,017 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 0464a2b21acb47289c7c51800d2b4b2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733201067142 2024-12-03T04:44:28,017 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0375c4b824874bcbbed8db339aaa1754, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733201067142 2024-12-03T04:44:28,026 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#B#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:28,026 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#A#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:28,027 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/e38d72ce7dc149a492f01ba2007bec58 is 50, key is test_row_0/A:col10/1733201067142/Put/seqid=0 2024-12-03T04:44:28,027 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/e964ab9f63374b1cbb13ee42055f3c13 is 50, key is test_row_0/B:col10/1733201067142/Put/seqid=0 2024-12-03T04:44:28,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742142_1318 (size=13425) 2024-12-03T04:44:28,044 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/e38d72ce7dc149a492f01ba2007bec58 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/e38d72ce7dc149a492f01ba2007bec58 2024-12-03T04:44:28,050 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/A of c65c1da21ae3e3e43d67b7de8f358916 into e38d72ce7dc149a492f01ba2007bec58(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:28,050 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:28,050 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/A, priority=13, startTime=1733201068014; duration=0sec 2024-12-03T04:44:28,050 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:28,050 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:A 2024-12-03T04:44:28,050 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:28,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742143_1319 (size=13425) 2024-12-03T04:44:28,053 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:28,053 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): c65c1da21ae3e3e43d67b7de8f358916/C is initiating minor compaction (all files) 2024-12-03T04:44:28,053 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c65c1da21ae3e3e43d67b7de8f358916/C in TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:28,053 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/ee03bb50d9d84c4eab39881a29c6150e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/f9aef64138184e55a2bbbbb9476466cd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b0f2c7fcacb34d1bbcee0a2ac3de1c93] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp, totalSize=37.0 K 2024-12-03T04:44:28,054 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee03bb50d9d84c4eab39881a29c6150e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733201064267 2024-12-03T04:44:28,058 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9aef64138184e55a2bbbbb9476466cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1733201065334 2024-12-03T04:44:28,058 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0f2c7fcacb34d1bbcee0a2ac3de1c93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733201067142 2024-12-03T04:44:28,060 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/e964ab9f63374b1cbb13ee42055f3c13 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e964ab9f63374b1cbb13ee42055f3c13 2024-12-03T04:44:28,064 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/B of c65c1da21ae3e3e43d67b7de8f358916 into e964ab9f63374b1cbb13ee42055f3c13(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:28,065 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:28,065 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/B, priority=13, startTime=1733201068014; duration=0sec 2024-12-03T04:44:28,065 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:28,065 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:B 2024-12-03T04:44:28,066 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c65c1da21ae3e3e43d67b7de8f358916#C#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:28,067 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/becb81ce9b5a41368c2bdee1e3c6c256 is 50, key is test_row_0/C:col10/1733201067142/Put/seqid=0 2024-12-03T04:44:28,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742144_1320 (size=13391) 2024-12-03T04:44:28,079 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/becb81ce9b5a41368c2bdee1e3c6c256 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/becb81ce9b5a41368c2bdee1e3c6c256 2024-12-03T04:44:28,089 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c65c1da21ae3e3e43d67b7de8f358916/C of c65c1da21ae3e3e43d67b7de8f358916 into becb81ce9b5a41368c2bdee1e3c6c256(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:28,090 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c65c1da21ae3e3e43d67b7de8f358916: 2024-12-03T04:44:28,090 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., storeName=c65c1da21ae3e3e43d67b7de8f358916/C, priority=13, startTime=1733201068014; duration=0sec 2024-12-03T04:44:28,090 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:28,090 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c65c1da21ae3e3e43d67b7de8f358916:C 2024-12-03T04:44:28,463 INFO [Thread-1037 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1037. 2024-12-03T04:44:28,463 DEBUG [Thread-1037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:28,463 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:28,463 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:28,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:28,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T04:44:28,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-03T04:44:28,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 134 2024-12-03T04:44:28,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-12-03T04:44:28,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-03T04:44:28,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-12-03T04:44:28,463 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T04:44:28,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2471 2024-12-03T04:44:28,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2481 2024-12-03T04:44:28,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2493 2024-12-03T04:44:28,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2494 2024-12-03T04:44:28,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2449 2024-12-03T04:44:28,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T04:44:28,464 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T04:44:28,464 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T04:44:28,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:28,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:28,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:28,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T04:44:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T04:44:28,468 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201068468"}]},"ts":"1733201068468"} 2024-12-03T04:44:28,470 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T04:44:28,470 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T04:44:28,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T04:44:28,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c65c1da21ae3e3e43d67b7de8f358916, UNASSIGN}] 2024-12-03T04:44:28,473 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c65c1da21ae3e3e43d67b7de8f358916, UNASSIGN 2024-12-03T04:44:28,473 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=c65c1da21ae3e3e43d67b7de8f358916, regionState=CLOSING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:44:28,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c65c1da21ae3e3e43d67b7de8f358916, UNASSIGN because future has completed 2024-12-03T04:44:28,475 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T04:44:28,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:44:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T04:44:28,627 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:28,627 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T04:44:28,627 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing c65c1da21ae3e3e43d67b7de8f358916, disabling compactions & flushes 2024-12-03T04:44:28,627 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:28,627 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:28,627 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. after waiting 0 ms 2024-12-03T04:44:28,627 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:28,628 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing c65c1da21ae3e3e43d67b7de8f358916 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-03T04:44:28,628 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=A 2024-12-03T04:44:28,628 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:28,628 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=B 2024-12-03T04:44:28,628 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:28,628 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c65c1da21ae3e3e43d67b7de8f358916, store=C 2024-12-03T04:44:28,628 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:28,633 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f33432674c344224a2a1ad51b20dcb87 is 50, key is test_row_1/A:col10/1733201067992/Put/seqid=0 2024-12-03T04:44:28,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742145_1321 (size=9857) 2024-12-03T04:44:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T04:44:29,041 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f33432674c344224a2a1ad51b20dcb87 2024-12-03T04:44:29,050 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/7dfd803778384b79805351a4fc385111 is 50, key is test_row_1/B:col10/1733201067992/Put/seqid=0 2024-12-03T04:44:29,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742146_1322 (size=9857) 2024-12-03T04:44:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T04:44:29,460 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/7dfd803778384b79805351a4fc385111 2024-12-03T04:44:29,468 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/4c9926e03e544c8d807158b3dfe202bc is 50, key is test_row_1/C:col10/1733201067992/Put/seqid=0 2024-12-03T04:44:29,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742147_1323 (size=9857) 2024-12-03T04:44:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T04:44:29,874 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/4c9926e03e544c8d807158b3dfe202bc 2024-12-03T04:44:29,878 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/A/f33432674c344224a2a1ad51b20dcb87 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f33432674c344224a2a1ad51b20dcb87 2024-12-03T04:44:29,881 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f33432674c344224a2a1ad51b20dcb87, entries=100, sequenceid=518, filesize=9.6 K 2024-12-03T04:44:29,882 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/B/7dfd803778384b79805351a4fc385111 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/7dfd803778384b79805351a4fc385111 2024-12-03T04:44:29,885 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/7dfd803778384b79805351a4fc385111, entries=100, sequenceid=518, filesize=9.6 K 2024-12-03T04:44:29,885 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/.tmp/C/4c9926e03e544c8d807158b3dfe202bc as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/4c9926e03e544c8d807158b3dfe202bc 2024-12-03T04:44:29,888 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/4c9926e03e544c8d807158b3dfe202bc, entries=100, sequenceid=518, filesize=9.6 K 2024-12-03T04:44:29,889 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for c65c1da21ae3e3e43d67b7de8f358916 in 1262ms, sequenceid=518, compaction requested=false 2024-12-03T04:44:29,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/a2bf9a66db8e4531b206742a32f88fa5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d83e8e1fc6e84c2d95d8afb2d304b430, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d599791007cc4f1ebea4d15400fb8b5b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/7c87d7ade0394d0ab06b6ff752e1154f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f645df993c7e45debb503bb0814edab8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/598d2b149f744e4481fdaf94c54cd14d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c71c99a7b87243b29f2b594cdb86ea89, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/23aaf77f729e4746a942d1e5f233449f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f60027b5e1d643dea34d867e39a27180, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/244b5aa2f0834e78ad73d8e8934708af, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/185bbccbfeb64e5b8a8212b7c65afe4b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/5025fbbd0ed5428ebadf0e95f0b034d4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f24c5838c1014710becca50f98f908ff, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/02535456923d4ddc805cff64c2d8c29b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/34fa64f7269644a3b2236f733801beb6, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/01c410ea5a694e57bb44efe226400982, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/6e703d07233847eb85b963de34c8bae2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9604828d466f463390a7f0bb6974fe0c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c8c9e307eb2f4c6e9cfde1e31b1f6ecb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/4a49332a30f34fb19e99b1ef28324a50, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0aebba5368a940818b6904fa535f1fb4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d2a055ced57247a396c71757429b0de8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c8fd09d09a5b40efb791b811a48b39c3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/da28bafa0e8743b2a28fa4eea22ac116, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9c4a0a74696b45b6b6fcb4345c1ef75a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d7d22482549341e0b2de5b8d56c7616d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/5f3e8180a2294c57836fa43c8257ac65, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0a12626079674aaf9df48c8305f390ac, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f79d9fe42e4047daafe00063ae693ee9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/aebff568d4bf4290a87eedd60a848880, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d51a0e7d5b314876b76006900ac8232a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/281681c63e4d4c1aa685189f56257bff, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0375c4b824874bcbbed8db339aaa1754] to archive 2024-12-03T04:44:29,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:44:29,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/a2bf9a66db8e4531b206742a32f88fa5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/a2bf9a66db8e4531b206742a32f88fa5 2024-12-03T04:44:29,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d83e8e1fc6e84c2d95d8afb2d304b430 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d83e8e1fc6e84c2d95d8afb2d304b430 2024-12-03T04:44:29,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d599791007cc4f1ebea4d15400fb8b5b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d599791007cc4f1ebea4d15400fb8b5b 2024-12-03T04:44:29,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/7c87d7ade0394d0ab06b6ff752e1154f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/7c87d7ade0394d0ab06b6ff752e1154f 2024-12-03T04:44:29,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f645df993c7e45debb503bb0814edab8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f645df993c7e45debb503bb0814edab8 2024-12-03T04:44:29,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/598d2b149f744e4481fdaf94c54cd14d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/598d2b149f744e4481fdaf94c54cd14d 2024-12-03T04:44:29,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c71c99a7b87243b29f2b594cdb86ea89 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c71c99a7b87243b29f2b594cdb86ea89 2024-12-03T04:44:29,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/23aaf77f729e4746a942d1e5f233449f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/23aaf77f729e4746a942d1e5f233449f 2024-12-03T04:44:29,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f60027b5e1d643dea34d867e39a27180 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f60027b5e1d643dea34d867e39a27180 2024-12-03T04:44:29,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/244b5aa2f0834e78ad73d8e8934708af to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/244b5aa2f0834e78ad73d8e8934708af 2024-12-03T04:44:29,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/185bbccbfeb64e5b8a8212b7c65afe4b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/185bbccbfeb64e5b8a8212b7c65afe4b 2024-12-03T04:44:29,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/5025fbbd0ed5428ebadf0e95f0b034d4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/5025fbbd0ed5428ebadf0e95f0b034d4 2024-12-03T04:44:29,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f24c5838c1014710becca50f98f908ff to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f24c5838c1014710becca50f98f908ff 2024-12-03T04:44:29,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/02535456923d4ddc805cff64c2d8c29b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/02535456923d4ddc805cff64c2d8c29b 2024-12-03T04:44:29,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/34fa64f7269644a3b2236f733801beb6 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/34fa64f7269644a3b2236f733801beb6 2024-12-03T04:44:29,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/01c410ea5a694e57bb44efe226400982 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/01c410ea5a694e57bb44efe226400982 2024-12-03T04:44:29,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/6e703d07233847eb85b963de34c8bae2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/6e703d07233847eb85b963de34c8bae2 2024-12-03T04:44:29,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9604828d466f463390a7f0bb6974fe0c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9604828d466f463390a7f0bb6974fe0c 2024-12-03T04:44:29,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c8c9e307eb2f4c6e9cfde1e31b1f6ecb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c8c9e307eb2f4c6e9cfde1e31b1f6ecb 2024-12-03T04:44:29,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/4a49332a30f34fb19e99b1ef28324a50 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/4a49332a30f34fb19e99b1ef28324a50 2024-12-03T04:44:29,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0aebba5368a940818b6904fa535f1fb4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0aebba5368a940818b6904fa535f1fb4 2024-12-03T04:44:29,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d2a055ced57247a396c71757429b0de8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d2a055ced57247a396c71757429b0de8 2024-12-03T04:44:29,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c8fd09d09a5b40efb791b811a48b39c3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/c8fd09d09a5b40efb791b811a48b39c3 2024-12-03T04:44:29,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/da28bafa0e8743b2a28fa4eea22ac116 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/da28bafa0e8743b2a28fa4eea22ac116 2024-12-03T04:44:29,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9c4a0a74696b45b6b6fcb4345c1ef75a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/9c4a0a74696b45b6b6fcb4345c1ef75a 2024-12-03T04:44:29,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d7d22482549341e0b2de5b8d56c7616d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d7d22482549341e0b2de5b8d56c7616d 2024-12-03T04:44:29,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/5f3e8180a2294c57836fa43c8257ac65 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/5f3e8180a2294c57836fa43c8257ac65 2024-12-03T04:44:29,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0a12626079674aaf9df48c8305f390ac to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0a12626079674aaf9df48c8305f390ac 2024-12-03T04:44:29,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f79d9fe42e4047daafe00063ae693ee9 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f79d9fe42e4047daafe00063ae693ee9 2024-12-03T04:44:29,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/aebff568d4bf4290a87eedd60a848880 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/aebff568d4bf4290a87eedd60a848880 2024-12-03T04:44:29,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d51a0e7d5b314876b76006900ac8232a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/d51a0e7d5b314876b76006900ac8232a 2024-12-03T04:44:29,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/281681c63e4d4c1aa685189f56257bff to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/281681c63e4d4c1aa685189f56257bff 2024-12-03T04:44:29,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0375c4b824874bcbbed8db339aaa1754 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/0375c4b824874bcbbed8db339aaa1754 2024-12-03T04:44:29,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/1a38e45419594b3c93fc82a03c3d290f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/661efa37cca14de8893d31d92a60c4d2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/c943a113772744c6951cfe2008353f96, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/fa0286b2e31d4ab5b5afcb5cb9780188, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/2359705d157247818bdf8f008b61374d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/595b7db7797642628bc5a0d187f5f0f1, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b7a44e5b836e4d2797145bd88dfb349b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0df4fb73dee143828ec8f487d4c2844c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f85b8d81598b46cf92302fd5f8cfb866, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/278df0920d674f73ad606c03a4df22c5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0e5c9c7a539141ac9ad73f076e52c93a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/bbc8f0aab20f4450a14bd260afd4758b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/c09fc02c13da40eb9dffe725fa19862d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f4903997a0fe4482b8a0eaa6fd3f66ac, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/eda9601d13b04b2fb9249535969293aa, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e0fb7b299f774c00be3442aef2b146ab, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/7a4023c0f0fb4c3c891665aaaf934ead, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b252ab4a4c384771b5ca16f115964518, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b823cc143e0a4205b100c280bed62350, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/72177cf0756a4a608db3bb2d70cd7cbf, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e381dd6cd6254117b7675e9d4303fbac, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/31f778592aac469c824e36d813408478, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/59b96bcb3fff49bc8317c1e406550bf8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ec5d6e6bc67949b7b31e46c86c424098, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ebd8924d37c24bdeadd5bb30d6bbeefc, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ce1d04dc75b64831b74fb089f494edcc, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/66db99f30c894d179bed5c1a9a25b89b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/8f74deae3900461aabd9c1825d3c84cf, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/5ad87b5d50e54a41870035346de87e03, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f6861e1e51a14977b8b5bc6a16ad9033, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/bda04b18674c42b0848e07fdeed70798, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/4203c575c483418881e957e3c11a4004, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0464a2b21acb47289c7c51800d2b4b2b] to archive 2024-12-03T04:44:29,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:44:29,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/1a38e45419594b3c93fc82a03c3d290f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/1a38e45419594b3c93fc82a03c3d290f 2024-12-03T04:44:29,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/661efa37cca14de8893d31d92a60c4d2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/661efa37cca14de8893d31d92a60c4d2 2024-12-03T04:44:29,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/c943a113772744c6951cfe2008353f96 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/c943a113772744c6951cfe2008353f96 2024-12-03T04:44:29,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/fa0286b2e31d4ab5b5afcb5cb9780188 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/fa0286b2e31d4ab5b5afcb5cb9780188 2024-12-03T04:44:29,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/2359705d157247818bdf8f008b61374d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/2359705d157247818bdf8f008b61374d 2024-12-03T04:44:29,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/595b7db7797642628bc5a0d187f5f0f1 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/595b7db7797642628bc5a0d187f5f0f1 2024-12-03T04:44:29,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b7a44e5b836e4d2797145bd88dfb349b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b7a44e5b836e4d2797145bd88dfb349b 2024-12-03T04:44:29,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0df4fb73dee143828ec8f487d4c2844c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0df4fb73dee143828ec8f487d4c2844c 2024-12-03T04:44:29,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f85b8d81598b46cf92302fd5f8cfb866 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f85b8d81598b46cf92302fd5f8cfb866 2024-12-03T04:44:29,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/278df0920d674f73ad606c03a4df22c5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/278df0920d674f73ad606c03a4df22c5 2024-12-03T04:44:29,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0e5c9c7a539141ac9ad73f076e52c93a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0e5c9c7a539141ac9ad73f076e52c93a 2024-12-03T04:44:29,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/bbc8f0aab20f4450a14bd260afd4758b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/bbc8f0aab20f4450a14bd260afd4758b 2024-12-03T04:44:29,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/c09fc02c13da40eb9dffe725fa19862d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/c09fc02c13da40eb9dffe725fa19862d 2024-12-03T04:44:29,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f4903997a0fe4482b8a0eaa6fd3f66ac to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f4903997a0fe4482b8a0eaa6fd3f66ac 2024-12-03T04:44:29,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/eda9601d13b04b2fb9249535969293aa to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/eda9601d13b04b2fb9249535969293aa 2024-12-03T04:44:29,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e0fb7b299f774c00be3442aef2b146ab to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e0fb7b299f774c00be3442aef2b146ab 2024-12-03T04:44:29,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/7a4023c0f0fb4c3c891665aaaf934ead to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/7a4023c0f0fb4c3c891665aaaf934ead 2024-12-03T04:44:29,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b252ab4a4c384771b5ca16f115964518 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b252ab4a4c384771b5ca16f115964518 2024-12-03T04:44:29,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b823cc143e0a4205b100c280bed62350 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/b823cc143e0a4205b100c280bed62350 2024-12-03T04:44:29,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/72177cf0756a4a608db3bb2d70cd7cbf to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/72177cf0756a4a608db3bb2d70cd7cbf 2024-12-03T04:44:29,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e381dd6cd6254117b7675e9d4303fbac to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e381dd6cd6254117b7675e9d4303fbac 2024-12-03T04:44:29,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/31f778592aac469c824e36d813408478 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/31f778592aac469c824e36d813408478 2024-12-03T04:44:29,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/59b96bcb3fff49bc8317c1e406550bf8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/59b96bcb3fff49bc8317c1e406550bf8 2024-12-03T04:44:29,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ec5d6e6bc67949b7b31e46c86c424098 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ec5d6e6bc67949b7b31e46c86c424098 2024-12-03T04:44:29,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ebd8924d37c24bdeadd5bb30d6bbeefc to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ebd8924d37c24bdeadd5bb30d6bbeefc 2024-12-03T04:44:29,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ce1d04dc75b64831b74fb089f494edcc to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/ce1d04dc75b64831b74fb089f494edcc 2024-12-03T04:44:29,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/66db99f30c894d179bed5c1a9a25b89b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/66db99f30c894d179bed5c1a9a25b89b 2024-12-03T04:44:29,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/8f74deae3900461aabd9c1825d3c84cf to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/8f74deae3900461aabd9c1825d3c84cf 2024-12-03T04:44:29,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/5ad87b5d50e54a41870035346de87e03 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/5ad87b5d50e54a41870035346de87e03 2024-12-03T04:44:29,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f6861e1e51a14977b8b5bc6a16ad9033 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/f6861e1e51a14977b8b5bc6a16ad9033 2024-12-03T04:44:29,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/bda04b18674c42b0848e07fdeed70798 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/bda04b18674c42b0848e07fdeed70798 2024-12-03T04:44:29,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/4203c575c483418881e957e3c11a4004 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/4203c575c483418881e957e3c11a4004 2024-12-03T04:44:29,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0464a2b21acb47289c7c51800d2b4b2b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/0464a2b21acb47289c7c51800d2b4b2b 2024-12-03T04:44:29,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/8e3ebaae08a44a3e87406722bf5c96d1, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/be648e8702df4ed38da7d68cd9d19435, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/04d3046fb7d24cb4814f7a95dadd325f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/23e7944239784b839b674d5b69380b21, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/6cdd852b8f494c72a32c7fbdf3268997, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/92b2fcf2928c41bc82fd5a15d4107a21, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/08fe265e54f844579f5eabc02e2bc739, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a879f505a77449059769c8aca2a12c3f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d88c8e01c9134bb2b525ccc191ef5518, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d4ac8837e5c0409b80b1c8ccca8275b4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/240bb5eb6c87422d93f4c31934d13874, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/f296d3fd5d3b4ea385285f77a88d5839, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/9cb9036223dc40f0a907e7f75cac7a20, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/492aefac32e548ff9b9293d27b516c61, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/dd7fdf3eb3124b988c9943824b789512, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/32cbf6cace4f4dba9a9ef0d24910614c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d7afe7b7cdc94b55b90487e04fa225de, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/6e909990a9684c8eaf32b7eae4728805, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/913c8aa0a7a84e739cd3f6c8b8c0dd08, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/85204f329fe94546bce0a87ee5748101, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/1fbfea4281ed4f29b3db551e673d250b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b97365ce1a3e4ec29e26647909ba2b99, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/69cc1ba64f1d4bc38d6c25fad798fde7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/78e6375d1de34c05b1e1f4e20d16aaaa, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d48c3a851db84f9496cf215295ac3157, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a9e6b539513349c6a8dd1b9d9ff85748, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/90904f872c2b4bff9b37ef53e1b5a3d5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/c330d8a0019d468da29b79127d338728, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/ee03bb50d9d84c4eab39881a29c6150e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/1afaa5d1452949a1a293ee085b6f3905, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/f9aef64138184e55a2bbbbb9476466cd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b0f2c7fcacb34d1bbcee0a2ac3de1c93] to archive 2024-12-03T04:44:29,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:44:29,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/8e3ebaae08a44a3e87406722bf5c96d1 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/8e3ebaae08a44a3e87406722bf5c96d1 2024-12-03T04:44:29,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/be648e8702df4ed38da7d68cd9d19435 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/be648e8702df4ed38da7d68cd9d19435 2024-12-03T04:44:29,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/04d3046fb7d24cb4814f7a95dadd325f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/04d3046fb7d24cb4814f7a95dadd325f 2024-12-03T04:44:29,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/23e7944239784b839b674d5b69380b21 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/23e7944239784b839b674d5b69380b21 2024-12-03T04:44:29,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/6cdd852b8f494c72a32c7fbdf3268997 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/6cdd852b8f494c72a32c7fbdf3268997 2024-12-03T04:44:29,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/92b2fcf2928c41bc82fd5a15d4107a21 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/92b2fcf2928c41bc82fd5a15d4107a21 2024-12-03T04:44:29,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/08fe265e54f844579f5eabc02e2bc739 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/08fe265e54f844579f5eabc02e2bc739 2024-12-03T04:44:29,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a879f505a77449059769c8aca2a12c3f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a879f505a77449059769c8aca2a12c3f 2024-12-03T04:44:29,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d88c8e01c9134bb2b525ccc191ef5518 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d88c8e01c9134bb2b525ccc191ef5518 2024-12-03T04:44:29,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d4ac8837e5c0409b80b1c8ccca8275b4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d4ac8837e5c0409b80b1c8ccca8275b4 2024-12-03T04:44:29,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/240bb5eb6c87422d93f4c31934d13874 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/240bb5eb6c87422d93f4c31934d13874 2024-12-03T04:44:29,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/f296d3fd5d3b4ea385285f77a88d5839 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/f296d3fd5d3b4ea385285f77a88d5839 2024-12-03T04:44:29,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/9cb9036223dc40f0a907e7f75cac7a20 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/9cb9036223dc40f0a907e7f75cac7a20 2024-12-03T04:44:29,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/492aefac32e548ff9b9293d27b516c61 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/492aefac32e548ff9b9293d27b516c61 2024-12-03T04:44:29,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/dd7fdf3eb3124b988c9943824b789512 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/dd7fdf3eb3124b988c9943824b789512 2024-12-03T04:44:29,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/32cbf6cace4f4dba9a9ef0d24910614c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/32cbf6cace4f4dba9a9ef0d24910614c 2024-12-03T04:44:29,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d7afe7b7cdc94b55b90487e04fa225de to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d7afe7b7cdc94b55b90487e04fa225de 2024-12-03T04:44:29,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/6e909990a9684c8eaf32b7eae4728805 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/6e909990a9684c8eaf32b7eae4728805 2024-12-03T04:44:29,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/913c8aa0a7a84e739cd3f6c8b8c0dd08 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/913c8aa0a7a84e739cd3f6c8b8c0dd08 2024-12-03T04:44:29,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/85204f329fe94546bce0a87ee5748101 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/85204f329fe94546bce0a87ee5748101 2024-12-03T04:44:29,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/1fbfea4281ed4f29b3db551e673d250b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/1fbfea4281ed4f29b3db551e673d250b 2024-12-03T04:44:29,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b97365ce1a3e4ec29e26647909ba2b99 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b97365ce1a3e4ec29e26647909ba2b99 2024-12-03T04:44:29,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/69cc1ba64f1d4bc38d6c25fad798fde7 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/69cc1ba64f1d4bc38d6c25fad798fde7 2024-12-03T04:44:29,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/78e6375d1de34c05b1e1f4e20d16aaaa to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/78e6375d1de34c05b1e1f4e20d16aaaa 2024-12-03T04:44:29,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d48c3a851db84f9496cf215295ac3157 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/d48c3a851db84f9496cf215295ac3157 2024-12-03T04:44:29,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a9e6b539513349c6a8dd1b9d9ff85748 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/a9e6b539513349c6a8dd1b9d9ff85748 2024-12-03T04:44:29,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/90904f872c2b4bff9b37ef53e1b5a3d5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/90904f872c2b4bff9b37ef53e1b5a3d5 2024-12-03T04:44:29,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/c330d8a0019d468da29b79127d338728 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/c330d8a0019d468da29b79127d338728 2024-12-03T04:44:29,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/ee03bb50d9d84c4eab39881a29c6150e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/ee03bb50d9d84c4eab39881a29c6150e 2024-12-03T04:44:29,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/1afaa5d1452949a1a293ee085b6f3905 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/1afaa5d1452949a1a293ee085b6f3905 2024-12-03T04:44:29,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/f9aef64138184e55a2bbbbb9476466cd to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/f9aef64138184e55a2bbbbb9476466cd 2024-12-03T04:44:29,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b0f2c7fcacb34d1bbcee0a2ac3de1c93 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/b0f2c7fcacb34d1bbcee0a2ac3de1c93 2024-12-03T04:44:29,987 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/recovered.edits/521.seqid, newMaxSeqId=521, maxSeqId=1 2024-12-03T04:44:29,988 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. 2024-12-03T04:44:29,988 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for c65c1da21ae3e3e43d67b7de8f358916: Waiting for close lock at 1733201068627Running coprocessor pre-close hooks at 1733201068627Disabling compacts and flushes for region at 1733201068627Disabling writes for close at 1733201068627Obtaining lock to block concurrent updates at 1733201068628 (+1 ms)Preparing flush snapshotting stores in c65c1da21ae3e3e43d67b7de8f358916 at 1733201068628Finished memstore snapshotting TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733201068628Flushing stores of TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916. at 1733201068629 (+1 ms)Flushing c65c1da21ae3e3e43d67b7de8f358916/A: creating writer at 1733201068629Flushing c65c1da21ae3e3e43d67b7de8f358916/A: appending metadata at 1733201068632 (+3 ms)Flushing c65c1da21ae3e3e43d67b7de8f358916/A: closing flushed file at 1733201068632Flushing c65c1da21ae3e3e43d67b7de8f358916/B: creating writer at 1733201069046 (+414 ms)Flushing c65c1da21ae3e3e43d67b7de8f358916/B: appending metadata at 1733201069050 (+4 ms)Flushing c65c1da21ae3e3e43d67b7de8f358916/B: closing flushed file at 1733201069050Flushing c65c1da21ae3e3e43d67b7de8f358916/C: creating writer at 1733201069464 (+414 ms)Flushing c65c1da21ae3e3e43d67b7de8f358916/C: appending metadata at 1733201069467 (+3 ms)Flushing c65c1da21ae3e3e43d67b7de8f358916/C: closing flushed file at 1733201069467Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e331d86: reopening flushed file at 1733201069877 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@532d1bc7: reopening flushed file at 1733201069881 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c066269: reopening flushed file at 1733201069885 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for c65c1da21ae3e3e43d67b7de8f358916 in 1262ms, sequenceid=518, compaction requested=false at 1733201069889 (+4 ms)Writing region close event to WAL at 1733201069985 (+96 ms)Running coprocessor post-close hooks at 1733201069988 (+3 ms)Closed at 1733201069988 2024-12-03T04:44:29,990 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:29,991 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=c65c1da21ae3e3e43d67b7de8f358916, regionState=CLOSED 2024-12-03T04:44:29,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:44:29,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-03T04:44:29,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure c65c1da21ae3e3e43d67b7de8f358916, server=32beccd76b35,43193,1733200986730 in 1.5180 sec 2024-12-03T04:44:29,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-12-03T04:44:29,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c65c1da21ae3e3e43d67b7de8f358916, UNASSIGN in 1.5220 sec 2024-12-03T04:44:29,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-03T04:44:29,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5260 sec 2024-12-03T04:44:29,999 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201069998"}]},"ts":"1733201069998"} 2024-12-03T04:44:30,000 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T04:44:30,000 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T04:44:30,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5350 sec 2024-12-03T04:44:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T04:44:30,612 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:30,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T04:44:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:30,614 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T04:44:30,615 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:30,618 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:30,620 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/recovered.edits] 2024-12-03T04:44:30,622 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/e38d72ce7dc149a492f01ba2007bec58 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/e38d72ce7dc149a492f01ba2007bec58 2024-12-03T04:44:30,623 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f33432674c344224a2a1ad51b20dcb87 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/A/f33432674c344224a2a1ad51b20dcb87 2024-12-03T04:44:30,625 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/7dfd803778384b79805351a4fc385111 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/7dfd803778384b79805351a4fc385111 2024-12-03T04:44:30,626 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e964ab9f63374b1cbb13ee42055f3c13 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/B/e964ab9f63374b1cbb13ee42055f3c13 2024-12-03T04:44:30,628 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/4c9926e03e544c8d807158b3dfe202bc to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/4c9926e03e544c8d807158b3dfe202bc 2024-12-03T04:44:30,629 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/becb81ce9b5a41368c2bdee1e3c6c256 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/C/becb81ce9b5a41368c2bdee1e3c6c256 2024-12-03T04:44:30,632 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/recovered.edits/521.seqid to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916/recovered.edits/521.seqid 2024-12-03T04:44:30,632 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/c65c1da21ae3e3e43d67b7de8f358916 2024-12-03T04:44:30,632 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T04:44:30,634 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:30,637 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T04:44:30,638 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T04:44:30,639 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:30,639 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T04:44:30,639 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733201070639"}]},"ts":"9223372036854775807"} 2024-12-03T04:44:30,641 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T04:44:30,641 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c65c1da21ae3e3e43d67b7de8f358916, NAME => 'TestAcidGuarantees,,1733201045160.c65c1da21ae3e3e43d67b7de8f358916.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T04:44:30,641 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T04:44:30,641 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733201070641"}]},"ts":"9223372036854775807"} 2024-12-03T04:44:30,643 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T04:44:30,643 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:30,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-12-03T04:44:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T04:44:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T04:44:30,722 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:30,731 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1456175902_22 at /127.0.0.1:47228 [Waiting for operation #206] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1704177745_22 at /127.0.0.1:38662 [Waiting for operation #432] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1456175902_22 at /127.0.0.1:56596 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1704177745_22 at /127.0.0.1:56580 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=834 (was 746) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4107 (was 3630) - AvailableMemoryMB LEAK? - 2024-12-03T04:44:30,741 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=834, ProcessCount=11, AvailableMemoryMB=4106 2024-12-03T04:44:30,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T04:44:30,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:44:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:30,744 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T04:44:30,745 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:30,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-12-03T04:44:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-03T04:44:30,745 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T04:44:30,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742148_1324 (size=1061) 2024-12-03T04:44:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-03T04:44:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-03T04:44:31,153 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676 2024-12-03T04:44:31,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742149_1325 (size=53) 2024-12-03T04:44:31,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:44:31,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8eeca2704febb93c298048b6df24ffb0, disabling compactions & flushes 2024-12-03T04:44:31,158 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:31,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:31,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. after waiting 0 ms 2024-12-03T04:44:31,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:31,158 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:31,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8eeca2704febb93c298048b6df24ffb0: Waiting for close lock at 1733201071158Disabling compacts and flushes for region at 1733201071158Disabling writes for close at 1733201071158Writing region close event to WAL at 1733201071158Closed at 1733201071158 2024-12-03T04:44:31,159 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T04:44:31,160 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733201071159"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733201071159"}]},"ts":"1733201071159"} 2024-12-03T04:44:31,161 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T04:44:31,162 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T04:44:31,162 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201071162"}]},"ts":"1733201071162"} 2024-12-03T04:44:31,163 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T04:44:31,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, ASSIGN}] 2024-12-03T04:44:31,165 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, ASSIGN 2024-12-03T04:44:31,165 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, ASSIGN; state=OFFLINE, location=32beccd76b35,43193,1733200986730; forceNewPlan=false, retain=false 2024-12-03T04:44:31,316 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=8eeca2704febb93c298048b6df24ffb0, regionState=OPENING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:44:31,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, ASSIGN because future has completed 2024-12-03T04:44:31,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:44:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-03T04:44:31,472 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:31,472 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:44:31,473 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,473 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:44:31,473 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,473 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,474 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,475 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:44:31,475 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eeca2704febb93c298048b6df24ffb0 columnFamilyName A 2024-12-03T04:44:31,475 DEBUG [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:31,475 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(327): Store=8eeca2704febb93c298048b6df24ffb0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:44:31,476 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,476 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:44:31,477 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eeca2704febb93c298048b6df24ffb0 columnFamilyName B 2024-12-03T04:44:31,477 DEBUG [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:31,477 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(327): Store=8eeca2704febb93c298048b6df24ffb0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:44:31,477 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,478 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:44:31,478 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eeca2704febb93c298048b6df24ffb0 columnFamilyName C 2024-12-03T04:44:31,478 DEBUG [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:31,478 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(327): Store=8eeca2704febb93c298048b6df24ffb0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:44:31,479 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:31,479 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,479 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,479 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,480 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,480 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,480 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:44:31,481 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,483 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T04:44:31,483 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 8eeca2704febb93c298048b6df24ffb0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70334080, jitterRate=0.04805946350097656}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:44:31,483 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:31,484 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 8eeca2704febb93c298048b6df24ffb0: Running coprocessor pre-open hook at 1733201071473Writing region info on filesystem at 1733201071473Initializing all the Stores at 1733201071473Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201071474 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201071474Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201071474Cleaning up temporary data from old regions at 1733201071480 (+6 ms)Running coprocessor post-open hooks at 1733201071483 (+3 ms)Region opened successfully at 1733201071484 (+1 ms) 2024-12-03T04:44:31,485 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., pid=81, masterSystemTime=1733201071469 2024-12-03T04:44:31,486 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:31,486 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:31,486 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=8eeca2704febb93c298048b6df24ffb0, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:44:31,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:44:31,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-03T04:44:31,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 in 170 msec 2024-12-03T04:44:31,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-03T04:44:31,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, ASSIGN in 326 msec 2024-12-03T04:44:31,492 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T04:44:31,493 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201071492"}]},"ts":"1733201071492"} 2024-12-03T04:44:31,494 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T04:44:31,494 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T04:44:31,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 751 msec 2024-12-03T04:44:31,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-03T04:44:31,882 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:31,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:31,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:31,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:31,884 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663b188, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:31,886 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6acf807a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:31,886 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:31,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:31,889 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T04:44:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,38397,1733200986071 2024-12-03T04:44:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@40c517d5 2024-12-03T04:44:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T04:44:31,890 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T04:44:31,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T04:44:31,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:44:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:31,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742150_1326 (size=1097) 2024-12-03T04:44:32,303 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T04:44:32,303 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T04:44:32,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T04:44:32,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, REOPEN/MOVE}] 2024-12-03T04:44:32,308 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, REOPEN/MOVE 2024-12-03T04:44:32,309 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=8eeca2704febb93c298048b6df24ffb0, regionState=CLOSING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:44:32,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, REOPEN/MOVE because future has completed 2024-12-03T04:44:32,310 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T04:44:32,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:44:32,462 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,462 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T04:44:32,462 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 8eeca2704febb93c298048b6df24ffb0, disabling compactions & flushes 2024-12-03T04:44:32,462 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:32,463 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:32,463 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. after waiting 0 ms 2024-12-03T04:44:32,463 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:32,466 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T04:44:32,467 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:32,467 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 8eeca2704febb93c298048b6df24ffb0: Waiting for close lock at 1733201072462Running coprocessor pre-close hooks at 1733201072462Disabling compacts and flushes for region at 1733201072462Disabling writes for close at 1733201072463 (+1 ms)Writing region close event to WAL at 1733201072463Running coprocessor post-close hooks at 1733201072467 (+4 ms)Closed at 1733201072467 2024-12-03T04:44:32,467 WARN [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: 8eeca2704febb93c298048b6df24ffb0 to self. 2024-12-03T04:44:32,469 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,469 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=8eeca2704febb93c298048b6df24ffb0, regionState=CLOSED 2024-12-03T04:44:32,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:44:32,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-03T04:44:32,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 in 161 msec 2024-12-03T04:44:32,473 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, REOPEN/MOVE; state=CLOSED, location=32beccd76b35,43193,1733200986730; forceNewPlan=false, retain=true 2024-12-03T04:44:32,624 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=8eeca2704febb93c298048b6df24ffb0, regionState=OPENING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:44:32,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, REOPEN/MOVE because future has completed 2024-12-03T04:44:32,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:44:32,780 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:32,780 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:44:32,781 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,781 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:44:32,781 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,781 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,782 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,783 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:44:32,783 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eeca2704febb93c298048b6df24ffb0 columnFamilyName A 2024-12-03T04:44:32,784 DEBUG [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:32,784 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(327): Store=8eeca2704febb93c298048b6df24ffb0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:44:32,785 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,785 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:44:32,786 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eeca2704febb93c298048b6df24ffb0 columnFamilyName B 2024-12-03T04:44:32,786 DEBUG [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:32,786 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(327): Store=8eeca2704febb93c298048b6df24ffb0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:44:32,786 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,787 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:44:32,787 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eeca2704febb93c298048b6df24ffb0 columnFamilyName C 2024-12-03T04:44:32,787 DEBUG [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:32,787 INFO [StoreOpener-8eeca2704febb93c298048b6df24ffb0-1 {}] regionserver.HStore(327): Store=8eeca2704febb93c298048b6df24ffb0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:44:32,787 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:32,787 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,788 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,789 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,789 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,789 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,790 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:44:32,791 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,791 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened 8eeca2704febb93c298048b6df24ffb0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67585562, jitterRate=0.00710335373878479}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:44:32,791 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,792 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for 8eeca2704febb93c298048b6df24ffb0: Running coprocessor pre-open hook at 1733201072781Writing region info on filesystem at 1733201072781Initializing all the Stores at 1733201072781Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201072782 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201072782Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201072782Cleaning up temporary data from old regions at 1733201072789 (+7 ms)Running coprocessor post-open hooks at 1733201072791 (+2 ms)Region opened successfully at 1733201072792 (+1 ms) 2024-12-03T04:44:32,792 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., pid=86, masterSystemTime=1733201072777 2024-12-03T04:44:32,794 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:32,794 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:32,794 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=8eeca2704febb93c298048b6df24ffb0, regionState=OPEN, openSeqNum=5, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:44:32,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:44:32,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-12-03T04:44:32,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 in 170 msec 2024-12-03T04:44:32,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-03T04:44:32,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, REOPEN/MOVE in 490 msec 2024-12-03T04:44:32,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-03T04:44:32,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-12-03T04:44:32,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-12-03T04:44:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-03T04:44:32,803 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:32,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:32,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:32,805 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:32,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:32,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7253443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:32,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:32,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:32,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132283a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:32,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:32,809 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70732bd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:32,810 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d524c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:32,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:32,812 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7330e30b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:32,813 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33fbebab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:32,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:32,815 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606ee4cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:32,816 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:32,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f972d94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:32,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:32,818 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ec4b348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:32,819 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f17576, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:32,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:32,821 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@202f6639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:32,822 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52942716, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:32,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:32,823 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7324d2c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:32,824 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:32,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a3b8a6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:32,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:32,826 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7482d7dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:32,827 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:32,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a932345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:32,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:32,829 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:32,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:32,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:32,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efa9435, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:32,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:32,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:32,830 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:32,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c41a0ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:44:32,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:44:32,831 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:44:32,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:44:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:44:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20765d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:44:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:44:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:32,832 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:44:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ad3939, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:44:32,833 DEBUG [Thread-1506 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:32,833 DEBUG [Thread-1504 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:32,833 DEBUG [Thread-1493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:32,834 DEBUG [Thread-1499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:32,834 DEBUG [Thread-1502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:32,835 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:32,835 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:32,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:32,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:32,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:32,837 DEBUG [Thread-1510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:32,837 DEBUG [Thread-1497 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:32,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:44:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:44:32,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:44:32,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:32,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:32,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:44:32,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:44:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:44:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-12-03T04:44:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T04:44:32,844 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:44:32,844 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:32,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:32,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:44:32,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:32,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:32,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:32,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:32,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:32,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:32,853 DEBUG [Thread-1491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:32,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:32,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:32,855 DEBUG [Thread-1495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:32,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:44:32,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:32,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:44:32,864 DEBUG [Thread-1508 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:44:32,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:44:32,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:44:32,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:44:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:44:32,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036e98a9d617794e33b2b98ed1e5e1b350_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201072851/Put/seqid=0 2024-12-03T04:44:32,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742151_1327 (size=14594) 2024-12-03T04:44:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T04:44:32,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733201072928, max timestamp=1733201072952Num uniques 859; 2024-12-03T04:44:32,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733201072928, max timestamp=1733201072952Num uniques 859; 2024-12-03T04:44:32,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T04:44:32,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T04:44:32,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733201072928, max timestamp=1733201072952Num uniques 859; 2024-12-03T04:44:32,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T04:44:32,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-03T04:44:32,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:32,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:32,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:32,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:32,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201133015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201133015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201133016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201133020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201133032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201133141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201133141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-03T04:44:33,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201133141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:33,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201133141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201133152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T04:44:33,292 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:33,296 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036e98a9d617794e33b2b98ed1e5e1b350_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036e98a9d617794e33b2b98ed1e5e1b350_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:33,303 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/cad7ef2ad753457d977af59ce1351658, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:33,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-03T04:44:33,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:33,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/cad7ef2ad753457d977af59ce1351658 is 175, key is test_row_0/A:col10/1733201072851/Put/seqid=0 2024-12-03T04:44:33,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742152_1328 (size=39549) 2024-12-03T04:44:33,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201133362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201133362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201133362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201133362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201133362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-03T04:44:33,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:33,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T04:44:33,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-03T04:44:33,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:33,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201133682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201133682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201133682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201133682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201133682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:33,716 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/cad7ef2ad753457d977af59ce1351658 2024-12-03T04:44:33,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/d300f6d1bc4d467186c290b6a16bcdd5 is 50, key is test_row_0/B:col10/1733201072851/Put/seqid=0 2024-12-03T04:44:33,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742153_1329 (size=12001) 2024-12-03T04:44:33,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-03T04:44:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-03T04:44:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:33,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T04:44:34,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-03T04:44:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:34,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:34,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/d300f6d1bc4d467186c290b6a16bcdd5 2024-12-03T04:44:34,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/74d8dfbbd1774ff08b31b50cf8d2e2ad is 50, key is test_row_0/C:col10/1733201072851/Put/seqid=0 2024-12-03T04:44:34,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742154_1330 (size=12001) 2024-12-03T04:44:34,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/74d8dfbbd1774ff08b31b50cf8d2e2ad 2024-12-03T04:44:34,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/cad7ef2ad753457d977af59ce1351658 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/cad7ef2ad753457d977af59ce1351658 2024-12-03T04:44:34,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/cad7ef2ad753457d977af59ce1351658, entries=200, sequenceid=14, filesize=38.6 K 2024-12-03T04:44:34,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/d300f6d1bc4d467186c290b6a16bcdd5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/d300f6d1bc4d467186c290b6a16bcdd5 2024-12-03T04:44:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201134202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:34,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201134202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:34,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201134202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:34,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201134204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:34,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:34,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:34,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:34,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:34,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:34,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:34,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201134206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:34,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/d300f6d1bc4d467186c290b6a16bcdd5, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T04:44:34,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:34,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:34,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:34,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/74d8dfbbd1774ff08b31b50cf8d2e2ad as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/74d8dfbbd1774ff08b31b50cf8d2e2ad 2024-12-03T04:44:34,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/74d8dfbbd1774ff08b31b50cf8d2e2ad, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T04:44:34,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8eeca2704febb93c298048b6df24ffb0 in 1366ms, sequenceid=14, compaction requested=false 2024-12-03T04:44:34,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:34,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-03T04:44:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:34,223 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-03T04:44:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:34,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ce29a56cd15244cd959673c217919118_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201073029/Put/seqid=0 2024-12-03T04:44:34,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742155_1331 (size=12154) 2024-12-03T04:44:34,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:34,650 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ce29a56cd15244cd959673c217919118_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ce29a56cd15244cd959673c217919118_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:34,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/005becf953584b75a73a789770618a04, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:34,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/005becf953584b75a73a789770618a04 is 175, key is test_row_0/A:col10/1733201073029/Put/seqid=0 2024-12-03T04:44:34,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742156_1332 (size=30955) 2024-12-03T04:44:34,657 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/005becf953584b75a73a789770618a04 2024-12-03T04:44:34,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/c8cd56e13071487885c90a89f09bc341 is 50, key is test_row_0/B:col10/1733201073029/Put/seqid=0 2024-12-03T04:44:34,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742157_1333 (size=12001) 2024-12-03T04:44:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T04:44:35,073 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/c8cd56e13071487885c90a89f09bc341 2024-12-03T04:44:35,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/f3a64acb25fe4b4093fca25e72595c65 is 50, key is test_row_0/C:col10/1733201073029/Put/seqid=0 2024-12-03T04:44:35,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742158_1334 (size=12001) 2024-12-03T04:44:35,163 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T04:44:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:35,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:35,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201135234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201135245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201135246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201135247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201135248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201135352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201135361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201135372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201135372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201135372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,494 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/f3a64acb25fe4b4093fca25e72595c65 2024-12-03T04:44:35,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/005becf953584b75a73a789770618a04 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/005becf953584b75a73a789770618a04 2024-12-03T04:44:35,503 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/005becf953584b75a73a789770618a04, entries=150, sequenceid=49, filesize=30.2 K 2024-12-03T04:44:35,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/c8cd56e13071487885c90a89f09bc341 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/c8cd56e13071487885c90a89f09bc341 2024-12-03T04:44:35,507 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/c8cd56e13071487885c90a89f09bc341, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T04:44:35,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/f3a64acb25fe4b4093fca25e72595c65 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f3a64acb25fe4b4093fca25e72595c65 2024-12-03T04:44:35,513 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f3a64acb25fe4b4093fca25e72595c65, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T04:44:35,514 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 8eeca2704febb93c298048b6df24ffb0 in 1291ms, sequenceid=49, compaction requested=false 2024-12-03T04:44:35,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:35,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:35,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-03T04:44:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-12-03T04:44:35,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-03T04:44:35,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6700 sec 2024-12-03T04:44:35,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 2.6820 sec 2024-12-03T04:44:35,531 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T04:44:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:35,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T04:44:35,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:35,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:35,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:35,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:35,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:35,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:35,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d9e23aa130a04bdab9ad7908fead7f22_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201075578/Put/seqid=0 2024-12-03T04:44:35,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742159_1335 (size=12154) 2024-12-03T04:44:35,657 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:35,661 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d9e23aa130a04bdab9ad7908fead7f22_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d9e23aa130a04bdab9ad7908fead7f22_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:35,662 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/db374f42bb3a4d53bc7810483c9c4498, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:35,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/db374f42bb3a4d53bc7810483c9c4498 is 175, key is test_row_0/A:col10/1733201075578/Put/seqid=0 2024-12-03T04:44:35,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742160_1336 (size=30955) 2024-12-03T04:44:35,679 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/db374f42bb3a4d53bc7810483c9c4498 2024-12-03T04:44:35,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201135668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201135668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/21b27757210648e3b9b7138d5c2581b5 is 50, key is test_row_0/B:col10/1733201075578/Put/seqid=0 2024-12-03T04:44:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201135677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201135682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742161_1337 (size=12001) 2024-12-03T04:44:35,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/21b27757210648e3b9b7138d5c2581b5 2024-12-03T04:44:35,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201135689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/3bbab6d8113d40b4a12a7cfbf017958d is 50, key is test_row_0/C:col10/1733201075578/Put/seqid=0 2024-12-03T04:44:35,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742162_1338 (size=12001) 2024-12-03T04:44:35,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201135792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201135792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201135801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201135801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:35,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201135825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:35,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:35,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201136012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201136012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201136021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201136021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201136042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/3bbab6d8113d40b4a12a7cfbf017958d 2024-12-03T04:44:36,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/db374f42bb3a4d53bc7810483c9c4498 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/db374f42bb3a4d53bc7810483c9c4498 2024-12-03T04:44:36,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/db374f42bb3a4d53bc7810483c9c4498, entries=150, sequenceid=63, filesize=30.2 K 2024-12-03T04:44:36,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/21b27757210648e3b9b7138d5c2581b5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/21b27757210648e3b9b7138d5c2581b5 2024-12-03T04:44:36,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/21b27757210648e3b9b7138d5c2581b5, entries=150, sequenceid=63, filesize=11.7 K 2024-12-03T04:44:36,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/3bbab6d8113d40b4a12a7cfbf017958d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/3bbab6d8113d40b4a12a7cfbf017958d 2024-12-03T04:44:36,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/3bbab6d8113d40b4a12a7cfbf017958d, entries=150, sequenceid=63, filesize=11.7 K 2024-12-03T04:44:36,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8eeca2704febb93c298048b6df24ffb0 in 568ms, sequenceid=63, compaction requested=true 2024-12-03T04:44:36,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:36,195 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:36,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:36,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:36,195 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:36,196 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/A is initiating minor compaction (all files) 2024-12-03T04:44:36,196 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/A in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:36,196 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/cad7ef2ad753457d977af59ce1351658, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/005becf953584b75a73a789770618a04, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/db374f42bb3a4d53bc7810483c9c4498] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=99.1 K 2024-12-03T04:44:36,196 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:36,196 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/cad7ef2ad753457d977af59ce1351658, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/005becf953584b75a73a789770618a04, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/db374f42bb3a4d53bc7810483c9c4498] 2024-12-03T04:44:36,196 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:36,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:36,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:36,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:36,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:36,197 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting cad7ef2ad753457d977af59ce1351658, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733201072841 2024-12-03T04:44:36,197 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 005becf953584b75a73a789770618a04, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733201072976 2024-12-03T04:44:36,198 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:36,198 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/B is initiating minor compaction (all files) 2024-12-03T04:44:36,198 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/B in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:36,198 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/d300f6d1bc4d467186c290b6a16bcdd5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/c8cd56e13071487885c90a89f09bc341, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/21b27757210648e3b9b7138d5c2581b5] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=35.2 K 2024-12-03T04:44:36,198 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting db374f42bb3a4d53bc7810483c9c4498, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733201075578 2024-12-03T04:44:36,198 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting d300f6d1bc4d467186c290b6a16bcdd5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733201072848 2024-12-03T04:44:36,199 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c8cd56e13071487885c90a89f09bc341, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733201072976 2024-12-03T04:44:36,199 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 21b27757210648e3b9b7138d5c2581b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733201075578 2024-12-03T04:44:36,207 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:36,208 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#B#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:36,209 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ef511c1d9dd74441b0b4ed90154bb929 is 50, key is test_row_0/B:col10/1733201075578/Put/seqid=0 2024-12-03T04:44:36,219 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120354e7a8829a314dc093385816be5956a2_8eeca2704febb93c298048b6df24ffb0 store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:36,221 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120354e7a8829a314dc093385816be5956a2_8eeca2704febb93c298048b6df24ffb0, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:36,221 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120354e7a8829a314dc093385816be5956a2_8eeca2704febb93c298048b6df24ffb0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:36,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742163_1339 (size=12104) 2024-12-03T04:44:36,239 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ef511c1d9dd74441b0b4ed90154bb929 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ef511c1d9dd74441b0b4ed90154bb929 2024-12-03T04:44:36,244 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/B of 8eeca2704febb93c298048b6df24ffb0 into ef511c1d9dd74441b0b4ed90154bb929(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:36,244 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:36,245 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/B, priority=13, startTime=1733201076196; duration=0sec 2024-12-03T04:44:36,245 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:36,245 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:B 2024-12-03T04:44:36,245 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:36,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742164_1340 (size=4469) 2024-12-03T04:44:36,247 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:36,247 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/C is initiating minor compaction (all files) 2024-12-03T04:44:36,247 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/C in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:36,247 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/74d8dfbbd1774ff08b31b50cf8d2e2ad, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f3a64acb25fe4b4093fca25e72595c65, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/3bbab6d8113d40b4a12a7cfbf017958d] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=35.2 K 2024-12-03T04:44:36,248 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 74d8dfbbd1774ff08b31b50cf8d2e2ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733201072848 2024-12-03T04:44:36,248 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#A#compaction#291 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:36,249 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/3afbcc9aef284929a6f479331d812485 is 175, key is test_row_0/A:col10/1733201075578/Put/seqid=0 2024-12-03T04:44:36,249 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting f3a64acb25fe4b4093fca25e72595c65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733201072976 2024-12-03T04:44:36,250 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bbab6d8113d40b4a12a7cfbf017958d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733201075578 2024-12-03T04:44:36,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742165_1341 (size=31058) 2024-12-03T04:44:36,274 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/3afbcc9aef284929a6f479331d812485 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3afbcc9aef284929a6f479331d812485 2024-12-03T04:44:36,286 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#C#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:36,287 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/0458758eff104277bf9296a5edc14847 is 50, key is test_row_0/C:col10/1733201075578/Put/seqid=0 2024-12-03T04:44:36,288 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/A of 8eeca2704febb93c298048b6df24ffb0 into 3afbcc9aef284929a6f479331d812485(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:36,288 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:36,288 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/A, priority=13, startTime=1733201076195; duration=0sec 2024-12-03T04:44:36,288 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:36,289 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:A 2024-12-03T04:44:36,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742166_1342 (size=12104) 2024-12-03T04:44:36,306 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/0458758eff104277bf9296a5edc14847 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/0458758eff104277bf9296a5edc14847 2024-12-03T04:44:36,311 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/C of 8eeca2704febb93c298048b6df24ffb0 into 0458758eff104277bf9296a5edc14847(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:36,311 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:36,311 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/C, priority=13, startTime=1733201076196; duration=0sec 2024-12-03T04:44:36,311 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:36,311 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:C 2024-12-03T04:44:36,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T04:44:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:36,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:36,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:36,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733201075660, max timestamp=1733201076338Num uniques 859; 2024-12-03T04:44:36,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:36,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733201075660, max timestamp=1733201076338Num uniques 859; 2024-12-03T04:44:36,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T04:44:36,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:36,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T04:44:36,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:36,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:36,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203130c6a51384f4e8e9615b222d6bcf933_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201076339/Put/seqid=0 2024-12-03T04:44:36,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742167_1343 (size=14594) 2024-12-03T04:44:36,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201136383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201136390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201136391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201136393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201136397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201136502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201136502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201136512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201136512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201136521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201136712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201136713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201136731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201136732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201136742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:36,770 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:36,789 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203130c6a51384f4e8e9615b222d6bcf933_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203130c6a51384f4e8e9615b222d6bcf933_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:36,794 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/a682417305b44ef49d4e76d0320a0ace, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:36,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/a682417305b44ef49d4e76d0320a0ace is 175, key is test_row_0/A:col10/1733201076339/Put/seqid=0 2024-12-03T04:44:36,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742168_1344 (size=39549) 2024-12-03T04:44:36,819 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/a682417305b44ef49d4e76d0320a0ace 2024-12-03T04:44:36,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/3eeae650995446098741e3b0e0e73a26 is 50, key is test_row_0/B:col10/1733201076339/Put/seqid=0 2024-12-03T04:44:36,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742169_1345 (size=12001) 2024-12-03T04:44:37,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T04:44:37,012 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:37,013 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:37,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-03T04:44:37,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T04:44:37,015 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:37,016 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:37,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:37,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201137031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:37,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201137031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:37,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201137042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201137042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:37,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201137061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:37,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T04:44:37,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T04:44:37,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:37,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:37,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:37,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.79 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/3eeae650995446098741e3b0e0e73a26 2024-12-03T04:44:37,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/98f807210abe461f995589fe1402c4ad is 50, key is test_row_0/C:col10/1733201076339/Put/seqid=0 2024-12-03T04:44:37,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742170_1346 (size=12001) 2024-12-03T04:44:37,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T04:44:37,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:37,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:37,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:37,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T04:44:37,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T04:44:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:37,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201137552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:37,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201137552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:37,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201137562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:37,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201137562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:37,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201137572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:37,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T04:44:37,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:37,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:37,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:37,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T04:44:37,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.79 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/98f807210abe461f995589fe1402c4ad 2024-12-03T04:44:37,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/a682417305b44ef49d4e76d0320a0ace as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a682417305b44ef49d4e76d0320a0ace 2024-12-03T04:44:37,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a682417305b44ef49d4e76d0320a0ace, entries=200, sequenceid=88, filesize=38.6 K 2024-12-03T04:44:37,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/3eeae650995446098741e3b0e0e73a26 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/3eeae650995446098741e3b0e0e73a26 2024-12-03T04:44:37,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/3eeae650995446098741e3b0e0e73a26, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T04:44:37,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/98f807210abe461f995589fe1402c4ad as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/98f807210abe461f995589fe1402c4ad 2024-12-03T04:44:37,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/98f807210abe461f995589fe1402c4ad, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T04:44:37,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.06 KB/65602, heapSize ~182.22 KB/186592, currentSize=114.05 KB/116790 for 8eeca2704febb93c298048b6df24ffb0 in 1362ms, sequenceid=88, compaction requested=false 2024-12-03T04:44:37,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:37,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T04:44:37,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:37,800 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:44:37,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:37,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:37,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:37,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:37,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:37,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039168efcf712840f8b1463f5c286e1646_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201076394/Put/seqid=0 2024-12-03T04:44:37,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742171_1347 (size=12154) 2024-12-03T04:44:37,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:37,827 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039168efcf712840f8b1463f5c286e1646_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039168efcf712840f8b1463f5c286e1646_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:37,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/3b7920fc9b924089ba3e3281af6b52fb, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:37,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/3b7920fc9b924089ba3e3281af6b52fb is 175, key is test_row_0/A:col10/1733201076394/Put/seqid=0 2024-12-03T04:44:37,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742172_1348 (size=30955) 2024-12-03T04:44:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T04:44:38,235 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/3b7920fc9b924089ba3e3281af6b52fb 2024-12-03T04:44:38,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ff6fdbce45714ef59938933bd45b2b4d is 50, key is test_row_0/B:col10/1733201076394/Put/seqid=0 2024-12-03T04:44:38,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742173_1349 (size=12001) 2024-12-03T04:44:38,256 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ff6fdbce45714ef59938933bd45b2b4d 2024-12-03T04:44:38,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/edb1edc719804b808d171c2bddf4c737 is 50, key is test_row_0/C:col10/1733201076394/Put/seqid=0 2024-12-03T04:44:38,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742174_1350 (size=12001) 2024-12-03T04:44:38,279 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/edb1edc719804b808d171c2bddf4c737 2024-12-03T04:44:38,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/3b7920fc9b924089ba3e3281af6b52fb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3b7920fc9b924089ba3e3281af6b52fb 2024-12-03T04:44:38,289 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3b7920fc9b924089ba3e3281af6b52fb, entries=150, sequenceid=108, filesize=30.2 K 2024-12-03T04:44:38,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ff6fdbce45714ef59938933bd45b2b4d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6fdbce45714ef59938933bd45b2b4d 2024-12-03T04:44:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,293 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6fdbce45714ef59938933bd45b2b4d, entries=150, sequenceid=108, filesize=11.7 K 2024-12-03T04:44:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/edb1edc719804b808d171c2bddf4c737 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/edb1edc719804b808d171c2bddf4c737 2024-12-03T04:44:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,298 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/edb1edc719804b808d171c2bddf4c737, entries=150, sequenceid=108, filesize=11.7 K 2024-12-03T04:44:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,299 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 8eeca2704febb93c298048b6df24ffb0 in 499ms, sequenceid=108, compaction requested=true 2024-12-03T04:44:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-03T04:44:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-03T04:44:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-03T04:44:38,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2850 sec 2024-12-03T04:44:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.2910 sec 2024-12-03T04:44:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:38,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:44:38,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:38,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:38,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:38,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:38,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:38,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120389ef0a345f5f4805bbab44f9592306c8_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201078709/Put/seqid=0 2024-12-03T04:44:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:38,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742175_1351 (size=21918) 2024-12-03T04:44:38,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733201078759, max timestamp=1733201078784Num uniques 859; 2024-12-03T04:44:38,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:44:38,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733201078759, max timestamp=1733201078784Num uniques 859; 2024-12-03T04:44:38,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:44:38,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733201078759, max timestamp=1733201078784Num uniques 859; 2024-12-03T04:44:38,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T04:44:38,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201138824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:38,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201138846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:38,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201138847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201138849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:38,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201138853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:38,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201138954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201138971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201138973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201138992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201138992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T04:44:39,161 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:39,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-03T04:44:39,164 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T04:44:39,165 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:39,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:39,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:39,184 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120389ef0a345f5f4805bbab44f9592306c8_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389ef0a345f5f4805bbab44f9592306c8_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:39,185 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/e1cbc6c239c840959cc5a5a9a8e5d8cb, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:39,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201139171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/e1cbc6c239c840959cc5a5a9a8e5d8cb is 175, key is test_row_0/A:col10/1733201078709/Put/seqid=0 2024-12-03T04:44:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742176_1352 (size=65323) 2024-12-03T04:44:39,194 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/e1cbc6c239c840959cc5a5a9a8e5d8cb 2024-12-03T04:44:39,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/486e6016174743668b88716f31828e3f is 50, key is test_row_0/B:col10/1733201078709/Put/seqid=0 2024-12-03T04:44:39,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201139192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201139192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742177_1353 (size=12001) 2024-12-03T04:44:39,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/486e6016174743668b88716f31828e3f 2024-12-03T04:44:39,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201139213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201139213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/24e850005a6b4fd09322a6dd41c06513 is 50, key is test_row_0/C:col10/1733201078709/Put/seqid=0 2024-12-03T04:44:39,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742178_1354 (size=12001) 2024-12-03T04:44:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T04:44:39,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/24e850005a6b4fd09322a6dd41c06513 2024-12-03T04:44:39,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/e1cbc6c239c840959cc5a5a9a8e5d8cb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e1cbc6c239c840959cc5a5a9a8e5d8cb 2024-12-03T04:44:39,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e1cbc6c239c840959cc5a5a9a8e5d8cb, entries=350, sequenceid=118, filesize=63.8 K 2024-12-03T04:44:39,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/486e6016174743668b88716f31828e3f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/486e6016174743668b88716f31828e3f 2024-12-03T04:44:39,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/486e6016174743668b88716f31828e3f, entries=150, sequenceid=118, filesize=11.7 K 2024-12-03T04:44:39,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/24e850005a6b4fd09322a6dd41c06513 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/24e850005a6b4fd09322a6dd41c06513 2024-12-03T04:44:39,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/24e850005a6b4fd09322a6dd41c06513, entries=150, sequenceid=118, filesize=11.7 K 2024-12-03T04:44:39,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for 8eeca2704febb93c298048b6df24ffb0 in 586ms, sequenceid=118, compaction requested=true 2024-12-03T04:44:39,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:39,298 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:44:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:39,298 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:44:39,299 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166885 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:44:39,299 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/A is initiating minor compaction (all files) 2024-12-03T04:44:39,299 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/A in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:39,299 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3afbcc9aef284929a6f479331d812485, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a682417305b44ef49d4e76d0320a0ace, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3b7920fc9b924089ba3e3281af6b52fb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e1cbc6c239c840959cc5a5a9a8e5d8cb] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=163.0 K 2024-12-03T04:44:39,299 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:39,299 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3afbcc9aef284929a6f479331d812485, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a682417305b44ef49d4e76d0320a0ace, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3b7920fc9b924089ba3e3281af6b52fb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e1cbc6c239c840959cc5a5a9a8e5d8cb] 2024-12-03T04:44:39,300 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3afbcc9aef284929a6f479331d812485, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733201075578 2024-12-03T04:44:39,300 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a682417305b44ef49d4e76d0320a0ace, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733201075687 2024-12-03T04:44:39,300 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b7920fc9b924089ba3e3281af6b52fb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733201076388 2024-12-03T04:44:39,301 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e1cbc6c239c840959cc5a5a9a8e5d8cb, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733201078635 2024-12-03T04:44:39,302 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:44:39,302 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/B is initiating minor compaction (all files) 2024-12-03T04:44:39,302 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/B in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:39,302 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ef511c1d9dd74441b0b4ed90154bb929, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/3eeae650995446098741e3b0e0e73a26, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6fdbce45714ef59938933bd45b2b4d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/486e6016174743668b88716f31828e3f] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=47.0 K 2024-12-03T04:44:39,306 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef511c1d9dd74441b0b4ed90154bb929, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733201075578 2024-12-03T04:44:39,306 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eeae650995446098741e3b0e0e73a26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733201075687 2024-12-03T04:44:39,307 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff6fdbce45714ef59938933bd45b2b4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733201076388 2024-12-03T04:44:39,307 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 486e6016174743668b88716f31828e3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733201078686 2024-12-03T04:44:39,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-03T04:44:39,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:39,319 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=138.65 KB heapSize=385.77 KB 2024-12-03T04:44:39,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:39,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:39,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:39,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:39,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:39,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:39,326 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:39,333 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#B#compaction#303 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:39,336 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/33c3e86f737748d287ebf15d7accf180 is 50, key is test_row_0/B:col10/1733201078709/Put/seqid=0 2024-12-03T04:44:39,356 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120336ca9718ce2245cca43a88168000629a_8eeca2704febb93c298048b6df24ffb0 store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:39,359 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120336ca9718ce2245cca43a88168000629a_8eeca2704febb93c298048b6df24ffb0, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:39,359 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120336ca9718ce2245cca43a88168000629a_8eeca2704febb93c298048b6df24ffb0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030878e9fe0c3c496a8d4db52179880fdb_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201078846/Put/seqid=0 2024-12-03T04:44:39,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742180_1356 (size=4469) 2024-12-03T04:44:39,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742179_1355 (size=12241) 2024-12-03T04:44:39,395 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/33c3e86f737748d287ebf15d7accf180 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/33c3e86f737748d287ebf15d7accf180 2024-12-03T04:44:39,408 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/B of 8eeca2704febb93c298048b6df24ffb0 into 33c3e86f737748d287ebf15d7accf180(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:39,408 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:39,408 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/B, priority=12, startTime=1733201079298; duration=0sec 2024-12-03T04:44:39,408 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:39,408 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:B 2024-12-03T04:44:39,409 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:44:39,410 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:44:39,410 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/C is initiating minor compaction (all files) 2024-12-03T04:44:39,410 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/C in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:39,410 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/0458758eff104277bf9296a5edc14847, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/98f807210abe461f995589fe1402c4ad, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/edb1edc719804b808d171c2bddf4c737, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/24e850005a6b4fd09322a6dd41c06513] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=47.0 K 2024-12-03T04:44:39,410 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0458758eff104277bf9296a5edc14847, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733201075578 2024-12-03T04:44:39,411 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98f807210abe461f995589fe1402c4ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733201075687 2024-12-03T04:44:39,411 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting edb1edc719804b808d171c2bddf4c737, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733201076388 2024-12-03T04:44:39,411 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e850005a6b4fd09322a6dd41c06513, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733201078686 2024-12-03T04:44:39,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742181_1357 (size=12304) 2024-12-03T04:44:39,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:39,427 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030878e9fe0c3c496a8d4db52179880fdb_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030878e9fe0c3c496a8d4db52179880fdb_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:39,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/ffde27de0bad46e385a26ed05932bfbd, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:39,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/ffde27de0bad46e385a26ed05932bfbd is 175, key is test_row_0/A:col10/1733201078846/Put/seqid=0 2024-12-03T04:44:39,432 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#C#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:39,433 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/b4e4bdf6c05640f6966196cfaff7ef0d is 50, key is test_row_0/C:col10/1733201078709/Put/seqid=0 2024-12-03T04:44:39,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742182_1358 (size=31105) 2024-12-03T04:44:39,444 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/ffde27de0bad46e385a26ed05932bfbd 2024-12-03T04:44:39,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/44020fba902f45008220f6033e2fc172 is 50, key is test_row_0/B:col10/1733201078846/Put/seqid=0 2024-12-03T04:44:39,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742183_1359 (size=12241) 2024-12-03T04:44:39,475 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/b4e4bdf6c05640f6966196cfaff7ef0d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/b4e4bdf6c05640f6966196cfaff7ef0d 2024-12-03T04:44:39,480 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/C of 8eeca2704febb93c298048b6df24ffb0 into b4e4bdf6c05640f6966196cfaff7ef0d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:39,480 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:39,480 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/C, priority=12, startTime=1733201079298; duration=0sec 2024-12-03T04:44:39,480 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:39,480 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:C 2024-12-03T04:44:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T04:44:39,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742184_1360 (size=12151) 2024-12-03T04:44:39,488 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/44020fba902f45008220f6033e2fc172 2024-12-03T04:44:39,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/d7d1a8c84ec9417caa49dfb8ed78cd8c is 50, key is test_row_0/C:col10/1733201078846/Put/seqid=0 2024-12-03T04:44:39,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742185_1361 (size=12151) 2024-12-03T04:44:39,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:39,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201139535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201139535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201139543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201139544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201139545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201139652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201139661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201139661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201139662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201139672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,780 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#A#compaction#302 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:39,781 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/ce1aeaad9db342d5a080d11b4a96bf1d is 175, key is test_row_0/A:col10/1733201078709/Put/seqid=0 2024-12-03T04:44:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T04:44:39,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742186_1362 (size=31195) 2024-12-03T04:44:39,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201139872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201139872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201139872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201139872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201139891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:39,915 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/d7d1a8c84ec9417caa49dfb8ed78cd8c 2024-12-03T04:44:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/ffde27de0bad46e385a26ed05932bfbd as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ffde27de0bad46e385a26ed05932bfbd 2024-12-03T04:44:39,928 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ffde27de0bad46e385a26ed05932bfbd, entries=150, sequenceid=153, filesize=30.4 K 2024-12-03T04:44:39,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/44020fba902f45008220f6033e2fc172 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/44020fba902f45008220f6033e2fc172 2024-12-03T04:44:39,938 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/44020fba902f45008220f6033e2fc172, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T04:44:39,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/d7d1a8c84ec9417caa49dfb8ed78cd8c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/d7d1a8c84ec9417caa49dfb8ed78cd8c 2024-12-03T04:44:39,943 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/d7d1a8c84ec9417caa49dfb8ed78cd8c, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T04:44:39,944 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~385.72 KB/394976, currentSize=46.96 KB/48090 for 8eeca2704febb93c298048b6df24ffb0 in 625ms, sequenceid=153, compaction requested=false 2024-12-03T04:44:39,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:39,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:39,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-03T04:44:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-03T04:44:39,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-03T04:44:39,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 781 msec 2024-12-03T04:44:39,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 796 msec 2024-12-03T04:44:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:40,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T04:44:40,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:40,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:40,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:40,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:40,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:40,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:40,210 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/ce1aeaad9db342d5a080d11b4a96bf1d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ce1aeaad9db342d5a080d11b4a96bf1d 2024-12-03T04:44:40,215 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/A of 8eeca2704febb93c298048b6df24ffb0 into ce1aeaad9db342d5a080d11b4a96bf1d(size=30.5 K), total size for store is 60.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:40,215 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:40,215 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/A, priority=12, startTime=1733201079297; duration=0sec 2024-12-03T04:44:40,215 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:40,215 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:A 2024-12-03T04:44:40,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fb4af61eb1244a2884134f8ac8843bd6_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201079544/Put/seqid=0 2024-12-03T04:44:40,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742187_1363 (size=12304) 2024-12-03T04:44:40,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201140257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201140258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201140262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201140269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201140279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T04:44:40,304 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:40,304 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-03T04:44:40,307 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:40,308 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:40,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T04:44:40,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201140382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201140384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201140391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201140391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201140392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T04:44:40,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T04:44:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:40,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201140602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201140602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201140602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T04:44:40,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201140611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201140611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T04:44:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:40,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:40,669 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:40,674 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fb4af61eb1244a2884134f8ac8843bd6_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fb4af61eb1244a2884134f8ac8843bd6_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:40,675 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/07e4b2382ccd4257889cec756eca842f, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:40,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/07e4b2382ccd4257889cec756eca842f is 175, key is test_row_0/A:col10/1733201079544/Put/seqid=0 2024-12-03T04:44:40,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742188_1364 (size=31105) 2024-12-03T04:44:40,692 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/07e4b2382ccd4257889cec756eca842f 2024-12-03T04:44:40,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/e2795ba6207345fa9e8bfe931739d8b0 is 50, key is test_row_0/B:col10/1733201079544/Put/seqid=0 2024-12-03T04:44:40,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742189_1365 (size=12151) 2024-12-03T04:44:40,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/e2795ba6207345fa9e8bfe931739d8b0 2024-12-03T04:44:40,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/899195064281416a90123222f33aa47e is 50, key is test_row_0/C:col10/1733201079544/Put/seqid=0 2024-12-03T04:44:40,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742190_1366 (size=12151) 2024-12-03T04:44:40,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/899195064281416a90123222f33aa47e 2024-12-03T04:44:40,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/07e4b2382ccd4257889cec756eca842f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/07e4b2382ccd4257889cec756eca842f 2024-12-03T04:44:40,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/07e4b2382ccd4257889cec756eca842f, entries=150, sequenceid=167, filesize=30.4 K 2024-12-03T04:44:40,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/e2795ba6207345fa9e8bfe931739d8b0 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e2795ba6207345fa9e8bfe931739d8b0 2024-12-03T04:44:40,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e2795ba6207345fa9e8bfe931739d8b0, entries=150, sequenceid=167, filesize=11.9 K 2024-12-03T04:44:40,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/899195064281416a90123222f33aa47e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/899195064281416a90123222f33aa47e 2024-12-03T04:44:40,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/899195064281416a90123222f33aa47e, entries=150, sequenceid=167, filesize=11.9 K 2024-12-03T04:44:40,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8eeca2704febb93c298048b6df24ffb0 in 567ms, sequenceid=167, compaction requested=true 2024-12-03T04:44:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:40,764 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:40,764 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:40,765 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:40,765 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/A is initiating minor compaction (all files) 2024-12-03T04:44:40,765 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/A in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:40,765 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ce1aeaad9db342d5a080d11b4a96bf1d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ffde27de0bad46e385a26ed05932bfbd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/07e4b2382ccd4257889cec756eca842f] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=91.2 K 2024-12-03T04:44:40,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:40,765 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:40,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:40,765 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ce1aeaad9db342d5a080d11b4a96bf1d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ffde27de0bad46e385a26ed05932bfbd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/07e4b2382ccd4257889cec756eca842f] 2024-12-03T04:44:40,766 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:40,766 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/B is initiating minor compaction (all files) 2024-12-03T04:44:40,766 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/B in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:40,766 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/33c3e86f737748d287ebf15d7accf180, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/44020fba902f45008220f6033e2fc172, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e2795ba6207345fa9e8bfe931739d8b0] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=35.7 K 2024-12-03T04:44:40,766 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce1aeaad9db342d5a080d11b4a96bf1d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733201078686 2024-12-03T04:44:40,767 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c3e86f737748d287ebf15d7accf180, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733201078686 2024-12-03T04:44:40,767 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffde27de0bad46e385a26ed05932bfbd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733201078839 2024-12-03T04:44:40,767 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 44020fba902f45008220f6033e2fc172, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733201078839 2024-12-03T04:44:40,768 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e4b2382ccd4257889cec756eca842f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733201079544 2024-12-03T04:44:40,768 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e2795ba6207345fa9e8bfe931739d8b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733201079544 2024-12-03T04:44:40,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:40,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:40,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T04:44:40,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:40,777 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:44:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:40,795 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:40,801 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:40,802 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/4503d902947e4afea3d1ade7420d061e is 50, key is test_row_0/B:col10/1733201079544/Put/seqid=0 2024-12-03T04:44:40,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d462a945a2ed4c77acd393f0a50d5834_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201080277/Put/seqid=0 2024-12-03T04:44:40,809 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412036e91bb311f8b48b59d4c2f5a2e0f11db_8eeca2704febb93c298048b6df24ffb0 store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:40,811 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412036e91bb311f8b48b59d4c2f5a2e0f11db_8eeca2704febb93c298048b6df24ffb0, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:40,811 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036e91bb311f8b48b59d4c2f5a2e0f11db_8eeca2704febb93c298048b6df24ffb0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:40,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742191_1367 (size=12493) 2024-12-03T04:44:40,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742192_1368 (size=12304) 2024-12-03T04:44:40,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742193_1369 (size=4469) 2024-12-03T04:44:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:40,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T04:44:40,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201140938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201140939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201140940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201140941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201140943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201141065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201141065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201141065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201141066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201141071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,233 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/4503d902947e4afea3d1ade7420d061e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/4503d902947e4afea3d1ade7420d061e 2024-12-03T04:44:41,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:41,239 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d462a945a2ed4c77acd393f0a50d5834_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d462a945a2ed4c77acd393f0a50d5834_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:41,239 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/B of 8eeca2704febb93c298048b6df24ffb0 into 4503d902947e4afea3d1ade7420d061e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:41,239 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:41,239 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/B, priority=13, startTime=1733201080764; duration=0sec 2024-12-03T04:44:41,240 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:41,240 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:B 2024-12-03T04:44:41,240 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:41,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/5d3e6f34988341c4b57e7f58e59a8666, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:41,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/5d3e6f34988341c4b57e7f58e59a8666 is 175, key is test_row_0/A:col10/1733201080277/Put/seqid=0 2024-12-03T04:44:41,242 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:41,242 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/C is initiating minor compaction (all files) 2024-12-03T04:44:41,242 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/C in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:41,242 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/b4e4bdf6c05640f6966196cfaff7ef0d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/d7d1a8c84ec9417caa49dfb8ed78cd8c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/899195064281416a90123222f33aa47e] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=35.7 K 2024-12-03T04:44:41,242 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting b4e4bdf6c05640f6966196cfaff7ef0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733201078686 2024-12-03T04:44:41,243 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting d7d1a8c84ec9417caa49dfb8ed78cd8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733201078839 2024-12-03T04:44:41,243 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 899195064281416a90123222f33aa47e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733201079544 2024-12-03T04:44:41,243 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#A#compaction#311 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:41,244 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/2851515ff1c14bd9b9be9e5df40ea498 is 175, key is test_row_0/A:col10/1733201079544/Put/seqid=0 2024-12-03T04:44:41,255 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#C#compaction#314 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:41,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742194_1370 (size=31105) 2024-12-03T04:44:41,255 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/645fe4a6bef849d5b80a18125e89866f is 50, key is test_row_0/C:col10/1733201079544/Put/seqid=0 2024-12-03T04:44:41,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742195_1371 (size=31447) 2024-12-03T04:44:41,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742196_1372 (size=12493) 2024-12-03T04:44:41,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201141281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201141281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201141281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201141281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201141292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T04:44:41,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201141602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201141602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201141602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201141611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201141611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:41,656 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/5d3e6f34988341c4b57e7f58e59a8666 2024-12-03T04:44:41,671 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/2851515ff1c14bd9b9be9e5df40ea498 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/2851515ff1c14bd9b9be9e5df40ea498 2024-12-03T04:44:41,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/161fe4da64d34520b3c3537488f84822 is 50, key is test_row_0/B:col10/1733201080277/Put/seqid=0 2024-12-03T04:44:41,676 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/A of 8eeca2704febb93c298048b6df24ffb0 into 2851515ff1c14bd9b9be9e5df40ea498(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:41,676 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:41,676 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/A, priority=13, startTime=1733201080764; duration=0sec 2024-12-03T04:44:41,676 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:41,676 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:A 2024-12-03T04:44:41,685 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/645fe4a6bef849d5b80a18125e89866f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/645fe4a6bef849d5b80a18125e89866f 2024-12-03T04:44:41,689 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/C of 8eeca2704febb93c298048b6df24ffb0 into 645fe4a6bef849d5b80a18125e89866f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:41,689 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:41,689 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/C, priority=13, startTime=1733201080768; duration=0sec 2024-12-03T04:44:41,690 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:41,690 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:C 2024-12-03T04:44:41,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742197_1373 (size=12151) 2024-12-03T04:44:42,095 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/161fe4da64d34520b3c3537488f84822 2024-12-03T04:44:42,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/fa4a6c99347949499229680530ab20c8 is 50, key is test_row_0/C:col10/1733201080277/Put/seqid=0 2024-12-03T04:44:42,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742198_1374 (size=12151) 2024-12-03T04:44:42,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201142121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:42,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201142122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:42,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201142122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:42,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201142131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:42,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201142131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T04:44:42,512 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/fa4a6c99347949499229680530ab20c8 2024-12-03T04:44:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/5d3e6f34988341c4b57e7f58e59a8666 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5d3e6f34988341c4b57e7f58e59a8666 2024-12-03T04:44:42,523 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5d3e6f34988341c4b57e7f58e59a8666, entries=150, sequenceid=188, filesize=30.4 K 2024-12-03T04:44:42,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/161fe4da64d34520b3c3537488f84822 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/161fe4da64d34520b3c3537488f84822 2024-12-03T04:44:42,527 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/161fe4da64d34520b3c3537488f84822, entries=150, sequenceid=188, filesize=11.9 K 2024-12-03T04:44:42,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/fa4a6c99347949499229680530ab20c8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/fa4a6c99347949499229680530ab20c8 2024-12-03T04:44:42,531 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/fa4a6c99347949499229680530ab20c8, entries=150, sequenceid=188, filesize=11.9 K 2024-12-03T04:44:42,532 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8eeca2704febb93c298048b6df24ffb0 in 1755ms, sequenceid=188, compaction requested=false 2024-12-03T04:44:42,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:42,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:42,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-03T04:44:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-03T04:44:42,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-03T04:44:42,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2260 sec 2024-12-03T04:44:42,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.2320 sec 2024-12-03T04:44:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:43,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T04:44:43,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:43,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:43,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:43,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:43,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:43,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:43,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e539970562f84ce8b450c8551f596827_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201080941/Put/seqid=0 2024-12-03T04:44:43,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742199_1375 (size=14794) 2024-12-03T04:44:43,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201143178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201143179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201143180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201143190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201143192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201143301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201143301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201143302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201143312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201143312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201143521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201143522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201143522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201143522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201143531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,566 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:43,570 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e539970562f84ce8b450c8551f596827_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e539970562f84ce8b450c8551f596827_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:43,570 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/d0e1a8f4df304138b41c802e262f584c, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:43,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/d0e1a8f4df304138b41c802e262f584c is 175, key is test_row_0/A:col10/1733201080941/Put/seqid=0 2024-12-03T04:44:43,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742200_1376 (size=39749) 2024-12-03T04:44:43,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201143842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201143842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201143842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201143842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201143851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:43,977 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/d0e1a8f4df304138b41c802e262f584c 2024-12-03T04:44:43,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/85ef6a1c396646a88b3111d0273df30d is 50, key is test_row_0/B:col10/1733201080941/Put/seqid=0 2024-12-03T04:44:43,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742201_1377 (size=12151) 2024-12-03T04:44:43,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/85ef6a1c396646a88b3111d0273df30d 2024-12-03T04:44:44,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/392f5f14e71a4da391cc9cbde25e25a7 is 50, key is test_row_0/C:col10/1733201080941/Put/seqid=0 2024-12-03T04:44:44,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742202_1378 (size=12151) 2024-12-03T04:44:44,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/392f5f14e71a4da391cc9cbde25e25a7 2024-12-03T04:44:44,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/d0e1a8f4df304138b41c802e262f584c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/d0e1a8f4df304138b41c802e262f584c 2024-12-03T04:44:44,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/d0e1a8f4df304138b41c802e262f584c, entries=200, sequenceid=204, filesize=38.8 K 2024-12-03T04:44:44,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/85ef6a1c396646a88b3111d0273df30d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/85ef6a1c396646a88b3111d0273df30d 2024-12-03T04:44:44,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/85ef6a1c396646a88b3111d0273df30d, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T04:44:44,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/392f5f14e71a4da391cc9cbde25e25a7 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/392f5f14e71a4da391cc9cbde25e25a7 2024-12-03T04:44:44,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/392f5f14e71a4da391cc9cbde25e25a7, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T04:44:44,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8eeca2704febb93c298048b6df24ffb0 in 900ms, sequenceid=204, compaction requested=true 2024-12-03T04:44:44,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:44,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:44,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:44,047 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:44,047 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:44,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:44,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:44,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:44,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:44,048 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:44,048 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:44,048 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/B is initiating minor compaction (all files) 2024-12-03T04:44:44,049 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/A is initiating minor compaction (all files) 2024-12-03T04:44:44,049 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/B in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:44,049 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/A in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:44,049 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/4503d902947e4afea3d1ade7420d061e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/161fe4da64d34520b3c3537488f84822, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/85ef6a1c396646a88b3111d0273df30d] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=35.9 K 2024-12-03T04:44:44,049 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/2851515ff1c14bd9b9be9e5df40ea498, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5d3e6f34988341c4b57e7f58e59a8666, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/d0e1a8f4df304138b41c802e262f584c] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=99.9 K 2024-12-03T04:44:44,049 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:44,049 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/2851515ff1c14bd9b9be9e5df40ea498, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5d3e6f34988341c4b57e7f58e59a8666, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/d0e1a8f4df304138b41c802e262f584c] 2024-12-03T04:44:44,049 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 4503d902947e4afea3d1ade7420d061e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733201079544 2024-12-03T04:44:44,049 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2851515ff1c14bd9b9be9e5df40ea498, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733201079544 2024-12-03T04:44:44,051 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 161fe4da64d34520b3c3537488f84822, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733201080256 2024-12-03T04:44:44,051 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3e6f34988341c4b57e7f58e59a8666, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733201080256 2024-12-03T04:44:44,053 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 85ef6a1c396646a88b3111d0273df30d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733201080940 2024-12-03T04:44:44,053 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0e1a8f4df304138b41c802e262f584c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733201080939 2024-12-03T04:44:44,059 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#B#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:44,060 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/a9060c9dbf7e43e5b2bc17231ba54c82 is 50, key is test_row_0/B:col10/1733201080941/Put/seqid=0 2024-12-03T04:44:44,066 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:44,068 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203aa4dbcd298be4105bf6a3b8c30bb9985_8eeca2704febb93c298048b6df24ffb0 store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:44,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742203_1379 (size=12595) 2024-12-03T04:44:44,070 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203aa4dbcd298be4105bf6a3b8c30bb9985_8eeca2704febb93c298048b6df24ffb0, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:44,070 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203aa4dbcd298be4105bf6a3b8c30bb9985_8eeca2704febb93c298048b6df24ffb0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:44,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742204_1380 (size=4469) 2024-12-03T04:44:44,086 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#A#compaction#321 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:44,086 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/44d476075aa94ec99cb5fa8f74e7728f is 175, key is test_row_0/A:col10/1733201080941/Put/seqid=0 2024-12-03T04:44:44,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742205_1381 (size=31549) 2024-12-03T04:44:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:44,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733201083176, max timestamp=1733201084364Num uniques 859; 2024-12-03T04:44:44,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:44:44,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=84.98 KB heapSize=247.03 KB 2024-12-03T04:44:44,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733201083176, max timestamp=1733201084364Num uniques 859; 2024-12-03T04:44:44,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733201083176, max timestamp=1733201084364Num uniques 859; 2024-12-03T04:44:44,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T04:44:44,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T04:44:44,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:44,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:44,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:44,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:44,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:44,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:44,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203beee9b14bad94654b3c16994946f2010_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201084370/Put/seqid=0 2024-12-03T04:44:44,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742206_1382 (size=17284) 2024-12-03T04:44:44,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733201084392, max timestamp=1733201084442Num uniques 859; 2024-12-03T04:44:44,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:44:44,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733201084392, max timestamp=1733201084442Num uniques 859; 2024-12-03T04:44:44,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733201084392, max timestamp=1733201084442Num uniques 859; 2024-12-03T04:44:44,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:44:44,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T04:44:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T04:44:44,462 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:44,462 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-03T04:44:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T04:44:44,464 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:44,465 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:44,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:44,477 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/a9060c9dbf7e43e5b2bc17231ba54c82 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/a9060c9dbf7e43e5b2bc17231ba54c82 2024-12-03T04:44:44,485 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/B of 8eeca2704febb93c298048b6df24ffb0 into a9060c9dbf7e43e5b2bc17231ba54c82(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:44,485 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:44,485 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/B, priority=13, startTime=1733201084047; duration=0sec 2024-12-03T04:44:44,485 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:44,485 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:B 2024-12-03T04:44:44,485 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:44,490 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:44,490 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/C is initiating minor compaction (all files) 2024-12-03T04:44:44,490 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/C in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:44,490 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/645fe4a6bef849d5b80a18125e89866f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/fa4a6c99347949499229680530ab20c8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/392f5f14e71a4da391cc9cbde25e25a7] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=35.9 K 2024-12-03T04:44:44,490 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 645fe4a6bef849d5b80a18125e89866f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733201079544 2024-12-03T04:44:44,491 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting fa4a6c99347949499229680530ab20c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733201080256 2024-12-03T04:44:44,493 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 392f5f14e71a4da391cc9cbde25e25a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733201080940 2024-12-03T04:44:44,502 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#C#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:44,503 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/def02be04b28407281a3926a7285860e is 50, key is test_row_0/C:col10/1733201080941/Put/seqid=0 2024-12-03T04:44:44,505 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/44d476075aa94ec99cb5fa8f74e7728f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/44d476075aa94ec99cb5fa8f74e7728f 2024-12-03T04:44:44,511 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/A of 8eeca2704febb93c298048b6df24ffb0 into 44d476075aa94ec99cb5fa8f74e7728f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:44,511 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:44,511 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/A, priority=13, startTime=1733201084047; duration=0sec 2024-12-03T04:44:44,511 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:44,511 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:A 2024-12-03T04:44:44,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201144498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742207_1383 (size=12595) 2024-12-03T04:44:44,528 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/def02be04b28407281a3926a7285860e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/def02be04b28407281a3926a7285860e 2024-12-03T04:44:44,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201144504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201144516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,535 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/C of 8eeca2704febb93c298048b6df24ffb0 into def02be04b28407281a3926a7285860e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:44,535 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:44,535 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/C, priority=13, startTime=1733201084048; duration=0sec 2024-12-03T04:44:44,536 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:44,536 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:C 2024-12-03T04:44:44,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201144518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201144519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T04:44:44,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:44,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:44,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:44,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:44,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:44,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:44,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201144632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201144633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201144641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201144641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201144641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:44,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:44,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:44,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:44,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:44,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:44,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T04:44:44,803 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:44,806 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203beee9b14bad94654b3c16994946f2010_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203beee9b14bad94654b3c16994946f2010_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:44,807 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/7d12d9186aad4b7baade3627a2a32c13, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:44,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/7d12d9186aad4b7baade3627a2a32c13 is 175, key is test_row_0/A:col10/1733201084370/Put/seqid=0 2024-12-03T04:44:44,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742208_1384 (size=48389) 2024-12-03T04:44:44,813 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/7d12d9186aad4b7baade3627a2a32c13 2024-12-03T04:44:44,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/48d6e12038574e159b231e7ba16fdb30 is 50, key is test_row_0/B:col10/1733201084370/Put/seqid=0 2024-12-03T04:44:44,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742209_1385 (size=12151) 2024-12-03T04:44:44,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201144851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201144851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201144862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201144862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201144862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:44,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:44,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:44,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:44,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:44,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:44,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T04:44:45,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201145161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201145161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201145182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201145182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201145182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/48d6e12038574e159b231e7ba16fdb30 2024-12-03T04:44:45,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:45,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:45,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/756962a7a1d64a09b27afe633085b9d9 is 50, key is test_row_0/C:col10/1733201084370/Put/seqid=0 2024-12-03T04:44:45,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742210_1386 (size=12151) 2024-12-03T04:44:45,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:45,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:45,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:45,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:45,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T04:44:45,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/756962a7a1d64a09b27afe633085b9d9 2024-12-03T04:44:45,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/7d12d9186aad4b7baade3627a2a32c13 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7d12d9186aad4b7baade3627a2a32c13 2024-12-03T04:44:45,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7d12d9186aad4b7baade3627a2a32c13, entries=250, sequenceid=227, filesize=47.3 K 2024-12-03T04:44:45,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/48d6e12038574e159b231e7ba16fdb30 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/48d6e12038574e159b231e7ba16fdb30 2024-12-03T04:44:45,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/48d6e12038574e159b231e7ba16fdb30, entries=150, sequenceid=227, filesize=11.9 K 2024-12-03T04:44:45,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/756962a7a1d64a09b27afe633085b9d9 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/756962a7a1d64a09b27afe633085b9d9 2024-12-03T04:44:45,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/756962a7a1d64a09b27afe633085b9d9, entries=150, sequenceid=227, filesize=11.9 K 2024-12-03T04:44:45,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.70 KB/42700, heapSize ~109.28 KB/111904, currentSize=154.31 KB/158010 for 8eeca2704febb93c298048b6df24ffb0 in 1290ms, sequenceid=227, compaction requested=false 2024-12-03T04:44:45,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:45,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=161.02 KB heapSize=451.31 KB 2024-12-03T04:44:45,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:45,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:44:45,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:45,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:44:45,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:45,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T04:44:45,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:45,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:45,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b64c53a105de47429aa118fbc3dd17d1_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201084489/Put/seqid=0 2024-12-03T04:44:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201145701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201145701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201145705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742211_1387 (size=14994) 2024-12-03T04:44:45,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201145706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201145710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201145822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201145822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201145832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:45,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:45,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:45,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:45,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:45,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:45,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:46,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:46,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201146041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:46,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201146041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:46,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201146052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:46,114 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:46,118 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b64c53a105de47429aa118fbc3dd17d1_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b64c53a105de47429aa118fbc3dd17d1_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:46,118 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/5e19ffd0cb5a46b9ba50379bcb77cd82, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:46,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/5e19ffd0cb5a46b9ba50379bcb77cd82 is 175, key is test_row_0/A:col10/1733201084489/Put/seqid=0 2024-12-03T04:44:46,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742212_1388 (size=39949) 2024-12-03T04:44:46,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:46,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:46,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201146362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:46,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201146362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:46,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201146362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:46,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:46,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:46,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,528 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/5e19ffd0cb5a46b9ba50379bcb77cd82 2024-12-03T04:44:46,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/bf9eaa97c3f34818b27a10b2c0d27816 is 50, key is test_row_0/B:col10/1733201084489/Put/seqid=0 2024-12-03T04:44:46,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742213_1389 (size=12301) 2024-12-03T04:44:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T04:44:46,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:46,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:46,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201146722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:46,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201146722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:46,733 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 8eeca2704febb93c298048b6df24ffb0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:46,733 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in 8eeca2704febb93c298048b6df24ffb0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:46,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:46,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:46,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201146881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:46,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:46,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201146881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:46,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:46,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201146881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:46,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:46,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:46,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:46,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:46,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/bf9eaa97c3f34818b27a10b2c0d27816 2024-12-03T04:44:46,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/f7cea33a540e42418f34c92dde079c10 is 50, key is test_row_0/C:col10/1733201084489/Put/seqid=0 2024-12-03T04:44:46,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742214_1390 (size=12301) 2024-12-03T04:44:47,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:47,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:47,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:47,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:47,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:47,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:47,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:47,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:47,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:47,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:47,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:47,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:47,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/f7cea33a540e42418f34c92dde079c10 2024-12-03T04:44:47,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/5e19ffd0cb5a46b9ba50379bcb77cd82 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5e19ffd0cb5a46b9ba50379bcb77cd82 2024-12-03T04:44:47,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5e19ffd0cb5a46b9ba50379bcb77cd82, entries=200, sequenceid=268, filesize=39.0 K 2024-12-03T04:44:47,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/bf9eaa97c3f34818b27a10b2c0d27816 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/bf9eaa97c3f34818b27a10b2c0d27816 2024-12-03T04:44:47,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/bf9eaa97c3f34818b27a10b2c0d27816, entries=150, sequenceid=268, filesize=12.0 K 2024-12-03T04:44:47,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:47,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:47,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:47,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:47,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:47,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/f7cea33a540e42418f34c92dde079c10 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f7cea33a540e42418f34c92dde079c10 2024-12-03T04:44:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f7cea33a540e42418f34c92dde079c10, entries=150, sequenceid=268, filesize=12.0 K 2024-12-03T04:44:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.27 KB/462096, currentSize=26.84 KB/27480 for 8eeca2704febb93c298048b6df24ffb0 in 1712ms, sequenceid=268, compaction requested=true 2024-12-03T04:44:47,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,400 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:47,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:47,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:47,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:47,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T04:44:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,401 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119887 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,401 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/A is initiating minor compaction (all files) 2024-12-03T04:44:47,401 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:47,401 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/A in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,401 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/44d476075aa94ec99cb5fa8f74e7728f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7d12d9186aad4b7baade3627a2a32c13, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5e19ffd0cb5a46b9ba50379bcb77cd82] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=117.1 K 2024-12-03T04:44:47,401 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:47,401 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/44d476075aa94ec99cb5fa8f74e7728f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7d12d9186aad4b7baade3627a2a32c13, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5e19ffd0cb5a46b9ba50379bcb77cd82] 2024-12-03T04:44:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,402 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44d476075aa94ec99cb5fa8f74e7728f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733201080940 2024-12-03T04:44:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,402 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d12d9186aad4b7baade3627a2a32c13, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733201083189 2024-12-03T04:44:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,403 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:47,403 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/B is initiating minor compaction (all files) 2024-12-03T04:44:47,403 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e19ffd0cb5a46b9ba50379bcb77cd82, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733201084489 2024-12-03T04:44:47,403 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/B in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:47,403 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/a9060c9dbf7e43e5b2bc17231ba54c82, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/48d6e12038574e159b231e7ba16fdb30, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/bf9eaa97c3f34818b27a10b2c0d27816] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=36.2 K 2024-12-03T04:44:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,404 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a9060c9dbf7e43e5b2bc17231ba54c82, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733201080940 2024-12-03T04:44:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,404 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 48d6e12038574e159b231e7ba16fdb30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733201084364 2024-12-03T04:44:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,405 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting bf9eaa97c3f34818b27a10b2c0d27816, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733201084489 2024-12-03T04:44:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,413 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,419 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,420 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/398cb2be19ac44ada83da63e7906e2b8 is 50, key is test_row_0/B:col10/1733201084489/Put/seqid=0 2024-12-03T04:44:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,421 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203f3d34a133fc548e4b12bc384298cc7bb_8eeca2704febb93c298048b6df24ffb0 store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,423 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203f3d34a133fc548e4b12bc384298cc7bb_8eeca2704febb93c298048b6df24ffb0, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,423 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f3d34a133fc548e4b12bc384298cc7bb_8eeca2704febb93c298048b6df24ffb0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742215_1391 (size=12847) 2024-12-03T04:44:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742216_1392 (size=4469) 2024-12-03T04:44:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,441 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#A#compaction#329 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:47,442 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/a14b5ab5ae1648a8b9f13dcb5062392c is 175, key is test_row_0/A:col10/1733201084489/Put/seqid=0 2024-12-03T04:44:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742217_1393 (size=31801) 2024-12-03T04:44:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,459 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/a14b5ab5ae1648a8b9f13dcb5062392c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a14b5ab5ae1648a8b9f13dcb5062392c 2024-12-03T04:44:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,464 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/A of 8eeca2704febb93c298048b6df24ffb0 into a14b5ab5ae1648a8b9f13dcb5062392c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:47,464 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:47,464 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/A, priority=13, startTime=1733201087399; duration=0sec 2024-12-03T04:44:47,464 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:47,464 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:A 2024-12-03T04:44:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,464 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,466 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:47,466 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/C is initiating minor compaction (all files) 2024-12-03T04:44:47,466 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/C in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,466 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/def02be04b28407281a3926a7285860e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/756962a7a1d64a09b27afe633085b9d9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f7cea33a540e42418f34c92dde079c10] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=36.2 K 2024-12-03T04:44:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,467 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting def02be04b28407281a3926a7285860e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733201080940 2024-12-03T04:44:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,467 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 756962a7a1d64a09b27afe633085b9d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733201084364 2024-12-03T04:44:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,467 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7cea33a540e42418f34c92dde079c10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733201084489 2024-12-03T04:44:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,476 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#C#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,476 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/22b37c9b8ed1412fbf2f892d68c23d55 is 50, key is test_row_0/C:col10/1733201084489/Put/seqid=0 2024-12-03T04:44:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742218_1394 (size=12847) 2024-12-03T04:44:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,502 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/22b37c9b8ed1412fbf2f892d68c23d55 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/22b37c9b8ed1412fbf2f892d68c23d55 2024-12-03T04:44:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,509 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/C of 8eeca2704febb93c298048b6df24ffb0 into 22b37c9b8ed1412fbf2f892d68c23d55(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,509 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:47,509 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/C, priority=13, startTime=1733201087400; duration=0sec 2024-12-03T04:44:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,509 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:47,509 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:C 2024-12-03T04:44:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T04:44:47,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,546 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-03T04:44:47,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:47,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:47,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:47,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:47,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203042260e132634c9c9fea2ba287ee8379_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201085703/Put/seqid=0 2024-12-03T04:44:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742219_1395 (size=9914) 2024-12-03T04:44:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,565 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203042260e132634c9c9fea2ba287ee8379_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203042260e132634c9c9fea2ba287ee8379_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/e51eaaebaab5465fa392a930c9df12a2, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/e51eaaebaab5465fa392a930c9df12a2 is 175, key is test_row_0/A:col10/1733201085703/Put/seqid=0 2024-12-03T04:44:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742220_1396 (size=22561) 2024-12-03T04:44:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,834 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/398cb2be19ac44ada83da63e7906e2b8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/398cb2be19ac44ada83da63e7906e2b8 2024-12-03T04:44:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,840 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/B of 8eeca2704febb93c298048b6df24ffb0 into 398cb2be19ac44ada83da63e7906e2b8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:47,840 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,840 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/B, priority=13, startTime=1733201087400; duration=0sec 2024-12-03T04:44:47,840 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:47,840 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:B 2024-12-03T04:44:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:47,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:47,998 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/e51eaaebaab5465fa392a930c9df12a2 2024-12-03T04:44:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ae38edcfdb03494a95ce659b6c220196 is 50, key is test_row_0/B:col10/1733201085703/Put/seqid=0 2024-12-03T04:44:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733201088024, max timestamp=1733201088062Num uniques 859; 2024-12-03T04:44:48,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T04:44:48,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733201088024, max timestamp=1733201088062Num uniques 859; 2024-12-03T04:44:48,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T04:44:48,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733201088024, max timestamp=1733201088062Num uniques 859; 2024-12-03T04:44:48,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T04:44:48,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742221_1397 (size=9857) 2024-12-03T04:44:48,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733201088116, max timestamp=1733201088145Num uniques 1118; 2024-12-03T04:44:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201148135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T04:44:48,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733201088116, max timestamp=1733201088145Num uniques 1118; 2024-12-03T04:44:48,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733201088116, max timestamp=1733201088145Num uniques 1118; 2024-12-03T04:44:48,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T04:44:48,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T04:44:48,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733201088206, max timestamp=1733201088234Num uniques 1027; 2024-12-03T04:44:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T04:44:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733201088206, max timestamp=1733201088234Num uniques 1027; 2024-12-03T04:44:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733201088206, max timestamp=1733201088234Num uniques 1027; 2024-12-03T04:44:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T04:44:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T04:44:48,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733201088275, max timestamp=1733201088309Num uniques 1036; 2024-12-03T04:44:48,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733201088275, max timestamp=1733201088309Num uniques 1036; 2024-12-03T04:44:48,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T04:44:48,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T04:44:48,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733201088275, max timestamp=1733201088309Num uniques 1036; 2024-12-03T04:44:48,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T04:44:48,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733201088374, max timestamp=1733201088392Num uniques 1045; 2024-12-03T04:44:48,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-03T04:44:48,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733201088374, max timestamp=1733201088392Num uniques 1045; 2024-12-03T04:44:48,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733201088374, max timestamp=1733201088392Num uniques 1045; 2024-12-03T04:44:48,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-03T04:44:48,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-03T04:44:48,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201148453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201148458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733201088436, max timestamp=1733201088466Num uniques 1104; 2024-12-03T04:44:48,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-03T04:44:48,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733201088436, max timestamp=1733201088466Num uniques 1104; 2024-12-03T04:44:48,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-03T04:44:48,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733201088436, max timestamp=1733201088466Num uniques 1104; 2024-12-03T04:44:48,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-03T04:44:48,478 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ae38edcfdb03494a95ce659b6c220196 2024-12-03T04:44:48,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/81d80f1bd4304b5e83d0a5779d56d04c is 50, key is test_row_0/C:col10/1733201085703/Put/seqid=0 2024-12-03T04:44:48,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742222_1398 (size=9857) 2024-12-03T04:44:48,488 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/81d80f1bd4304b5e83d0a5779d56d04c 2024-12-03T04:44:48,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/e51eaaebaab5465fa392a930c9df12a2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e51eaaebaab5465fa392a930c9df12a2 2024-12-03T04:44:48,499 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e51eaaebaab5465fa392a930c9df12a2, entries=100, sequenceid=277, filesize=22.0 K 2024-12-03T04:44:48,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ae38edcfdb03494a95ce659b6c220196 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ae38edcfdb03494a95ce659b6c220196 2024-12-03T04:44:48,504 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ae38edcfdb03494a95ce659b6c220196, entries=100, sequenceid=277, filesize=9.6 K 2024-12-03T04:44:48,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/81d80f1bd4304b5e83d0a5779d56d04c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/81d80f1bd4304b5e83d0a5779d56d04c 2024-12-03T04:44:48,508 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/81d80f1bd4304b5e83d0a5779d56d04c, entries=100, sequenceid=277, filesize=9.6 K 2024-12-03T04:44:48,516 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=60.38 KB/61830 for 8eeca2704febb93c298048b6df24ffb0 in 970ms, sequenceid=277, compaction requested=false 2024-12-03T04:44:48,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:48,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:48,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-03T04:44:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-03T04:44:48,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=67.09 KB heapSize=186.23 KB 2024-12-03T04:44:48,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:48,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:48,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:48,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:48,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:48,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:48,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-03T04:44:48,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0550 sec 2024-12-03T04:44:48,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 4.0610 sec 2024-12-03T04:44:48,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203df15b08a69704b5a902c04fd762bc616_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201088477/Put/seqid=0 2024-12-03T04:44:48,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742223_1399 (size=14994) 2024-12-03T04:44:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T04:44:48,632 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:48,632 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-03T04:44:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T04:44:48,635 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:48,638 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:48,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:48,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201148630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201148631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201148644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T04:44:48,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201148752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201148752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201148753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,759 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 8eeca2704febb93c298048b6df24ffb0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:48,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201148761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,776 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in 8eeca2704febb93c298048b6df24ffb0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201148777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T04:44:48,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:48,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:48,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:48,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:48,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:48,943 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:48,946 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203df15b08a69704b5a902c04fd762bc616_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203df15b08a69704b5a902c04fd762bc616_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:48,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/29b65ea1ac0141b0b995dc2a5794f5ff, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:48,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/29b65ea1ac0141b0b995dc2a5794f5ff is 175, key is test_row_0/A:col10/1733201088477/Put/seqid=0 2024-12-03T04:44:48,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T04:44:48,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:48,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:48,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:48,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T04:44:48,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742224_1400 (size=39949) 2024-12-03T04:44:48,956 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/29b65ea1ac0141b0b995dc2a5794f5ff 2024-12-03T04:44:48,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/e3140715bd6b44d59fb55046ee2400b6 is 50, key is test_row_0/B:col10/1733201088477/Put/seqid=0 2024-12-03T04:44:48,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742225_1401 (size=12301) 2024-12-03T04:44:48,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201148971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201148971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201148981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:49,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T04:44:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T04:44:49,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:49,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T04:44:49,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201149291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:49,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201149291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:49,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201149291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:49,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/e3140715bd6b44d59fb55046ee2400b6 2024-12-03T04:44:49,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/b9483cdb4c45452bb1fe0f96dc0b78a8 is 50, key is test_row_0/C:col10/1733201088477/Put/seqid=0 2024-12-03T04:44:49,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742226_1402 (size=12301) 2024-12-03T04:44:49,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T04:44:49,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:49,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T04:44:49,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:49,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T04:44:49,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:49,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T04:44:49,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/b9483cdb4c45452bb1fe0f96dc0b78a8 2024-12-03T04:44:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/29b65ea1ac0141b0b995dc2a5794f5ff as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/29b65ea1ac0141b0b995dc2a5794f5ff 2024-12-03T04:44:49,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/29b65ea1ac0141b0b995dc2a5794f5ff, entries=200, sequenceid=391, filesize=39.0 K 2024-12-03T04:44:49,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/e3140715bd6b44d59fb55046ee2400b6 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e3140715bd6b44d59fb55046ee2400b6 2024-12-03T04:44:49,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e3140715bd6b44d59fb55046ee2400b6, entries=150, sequenceid=391, filesize=12.0 K 2024-12-03T04:44:49,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201149812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:49,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201149813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:49,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/b9483cdb4c45452bb1fe0f96dc0b78a8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/b9483cdb4c45452bb1fe0f96dc0b78a8 2024-12-03T04:44:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:49,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/b9483cdb4c45452bb1fe0f96dc0b78a8, entries=150, sequenceid=391, filesize=12.0 K 2024-12-03T04:44:49,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~186.19 KB/190656, currentSize=114.05 KB/116790 for 8eeca2704febb93c298048b6df24ffb0 in 1311ms, sequenceid=391, compaction requested=true 2024-12-03T04:44:49,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:49,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:49,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:49,830 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:49,830 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:49,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:49,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:49,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:49,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:49,831 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:49,831 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:49,831 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/A is initiating minor compaction (all files) 2024-12-03T04:44:49,831 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/B is initiating minor compaction (all files) 2024-12-03T04:44:49,831 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/B in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,831 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/A in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,831 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/398cb2be19ac44ada83da63e7906e2b8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ae38edcfdb03494a95ce659b6c220196, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e3140715bd6b44d59fb55046ee2400b6] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=34.2 K 2024-12-03T04:44:49,831 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a14b5ab5ae1648a8b9f13dcb5062392c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e51eaaebaab5465fa392a930c9df12a2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/29b65ea1ac0141b0b995dc2a5794f5ff] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=92.1 K 2024-12-03T04:44:49,831 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,831 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a14b5ab5ae1648a8b9f13dcb5062392c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e51eaaebaab5465fa392a930c9df12a2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/29b65ea1ac0141b0b995dc2a5794f5ff] 2024-12-03T04:44:49,831 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 398cb2be19ac44ada83da63e7906e2b8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733201084489 2024-12-03T04:44:49,832 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting a14b5ab5ae1648a8b9f13dcb5062392c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733201084489 2024-12-03T04:44:49,832 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ae38edcfdb03494a95ce659b6c220196, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733201085703 2024-12-03T04:44:49,832 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting e51eaaebaab5465fa392a930c9df12a2, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733201085703 2024-12-03T04:44:49,832 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e3140715bd6b44d59fb55046ee2400b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733201088477 2024-12-03T04:44:49,832 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b65ea1ac0141b0b995dc2a5794f5ff, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733201088477 2024-12-03T04:44:49,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201088623, max timestamp=1733201089823Num uniques 859; 2024-12-03T04:44:49,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:49,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201088623, max timestamp=1733201089823Num uniques 859; 2024-12-03T04:44:49,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:49,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201088623, max timestamp=1733201089823Num uniques 859; 2024-12-03T04:44:49,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:44:49,850 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#B#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:49,851 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/4215a661eec949c0bcea0d94eefd6004 is 50, key is test_row_0/B:col10/1733201088477/Put/seqid=0 2024-12-03T04:44:49,852 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:49,867 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203ef2902628b6c4d0eb8a2feb9f7df7250_8eeca2704febb93c298048b6df24ffb0 store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:49,870 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203ef2902628b6c4d0eb8a2feb9f7df7250_8eeca2704febb93c298048b6df24ffb0, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:49,870 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ef2902628b6c4d0eb8a2feb9f7df7250_8eeca2704febb93c298048b6df24ffb0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:49,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T04:44:49,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,872 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=46.96 KB heapSize=126 KB 2024-12-03T04:44:49,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:49,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:49,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:49,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:49,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:49,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:49,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:49,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742227_1403 (size=12949) 2024-12-03T04:44:49,892 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/4215a661eec949c0bcea0d94eefd6004 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/4215a661eec949c0bcea0d94eefd6004 2024-12-03T04:44:49,897 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/B of 8eeca2704febb93c298048b6df24ffb0 into 4215a661eec949c0bcea0d94eefd6004(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:49,897 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:49,897 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/B, priority=13, startTime=1733201089830; duration=0sec 2024-12-03T04:44:49,897 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:49,897 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:B 2024-12-03T04:44:49,897 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:49,904 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:49,904 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/C is initiating minor compaction (all files) 2024-12-03T04:44:49,904 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/C in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:49,904 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/22b37c9b8ed1412fbf2f892d68c23d55, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/81d80f1bd4304b5e83d0a5779d56d04c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/b9483cdb4c45452bb1fe0f96dc0b78a8] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=34.2 K 2024-12-03T04:44:49,905 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b37c9b8ed1412fbf2f892d68c23d55, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733201084489 2024-12-03T04:44:49,905 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d80f1bd4304b5e83d0a5779d56d04c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733201085703 2024-12-03T04:44:49,905 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting b9483cdb4c45452bb1fe0f96dc0b78a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733201088477 2024-12-03T04:44:49,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b06945648042445e887ea5959c44199b_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201089856/Put/seqid=0 2024-12-03T04:44:49,914 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:49,920 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/7ce87d96421f428d93c23daa612d065e is 50, key is test_row_0/C:col10/1733201088477/Put/seqid=0 2024-12-03T04:44:49,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742228_1404 (size=4469) 2024-12-03T04:44:49,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742229_1405 (size=12949) 2024-12-03T04:44:49,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742230_1406 (size=14994) 2024-12-03T04:44:50,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201090079, max timestamp=1733201090166Num uniques 859; 2024-12-03T04:44:50,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:44:50,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201090079, max timestamp=1733201090166Num uniques 859; 2024-12-03T04:44:50,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:44:50,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201090079, max timestamp=1733201090166Num uniques 859; 2024-12-03T04:44:50,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:44:50,357 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#A#compaction#339 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:50,357 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/e66185cb9d974f90ba45359e6afac7d0 is 175, key is test_row_0/A:col10/1733201088477/Put/seqid=0 2024-12-03T04:44:50,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742231_1407 (size=31903) 2024-12-03T04:44:50,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:50,367 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b06945648042445e887ea5959c44199b_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b06945648042445e887ea5959c44199b_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:50,368 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/7ce87d96421f428d93c23daa612d065e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/7ce87d96421f428d93c23daa612d065e 2024-12-03T04:44:50,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/06d94bb02d3f4782ad08b3fa53147c9a, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:50,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/06d94bb02d3f4782ad08b3fa53147c9a is 175, key is test_row_0/A:col10/1733201089856/Put/seqid=0 2024-12-03T04:44:50,371 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/C of 8eeca2704febb93c298048b6df24ffb0 into 7ce87d96421f428d93c23daa612d065e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:50,371 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:50,372 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/C, priority=13, startTime=1733201089831; duration=0sec 2024-12-03T04:44:50,372 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:50,372 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:C 2024-12-03T04:44:50,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742232_1408 (size=39949) 2024-12-03T04:44:50,372 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/06d94bb02d3f4782ad08b3fa53147c9a 2024-12-03T04:44:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ff6bd237dc774ba1b2ef6c4a3b8fc2be is 50, key is test_row_0/B:col10/1733201089856/Put/seqid=0 2024-12-03T04:44:50,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201150385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:50,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742233_1409 (size=12301) 2024-12-03T04:44:50,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201150491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:50,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201150712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:50,767 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/e66185cb9d974f90ba45359e6afac7d0 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e66185cb9d974f90ba45359e6afac7d0 2024-12-03T04:44:50,773 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/A of 8eeca2704febb93c298048b6df24ffb0 into e66185cb9d974f90ba45359e6afac7d0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:50,773 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:50,773 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/A, priority=13, startTime=1733201089830; duration=0sec 2024-12-03T04:44:50,773 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:50,773 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:A 2024-12-03T04:44:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T04:44:50,791 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ff6bd237dc774ba1b2ef6c4a3b8fc2be 2024-12-03T04:44:50,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/e777036201b5435a91eb26493a98f7e8 is 50, key is test_row_0/C:col10/1733201089856/Put/seqid=0 2024-12-03T04:44:50,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742234_1410 (size=12301) 2024-12-03T04:44:50,809 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/e777036201b5435a91eb26493a98f7e8 2024-12-03T04:44:50,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/06d94bb02d3f4782ad08b3fa53147c9a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/06d94bb02d3f4782ad08b3fa53147c9a 2024-12-03T04:44:50,820 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/06d94bb02d3f4782ad08b3fa53147c9a, entries=200, sequenceid=416, filesize=39.0 K 2024-12-03T04:44:50,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ff6bd237dc774ba1b2ef6c4a3b8fc2be as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6bd237dc774ba1b2ef6c4a3b8fc2be 2024-12-03T04:44:50,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201150822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:50,829 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in 8eeca2704febb93c298048b6df24ffb0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:50,833 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6bd237dc774ba1b2ef6c4a3b8fc2be, entries=150, sequenceid=416, filesize=12.0 K 2024-12-03T04:44:50,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/e777036201b5435a91eb26493a98f7e8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/e777036201b5435a91eb26493a98f7e8 2024-12-03T04:44:50,837 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/e777036201b5435a91eb26493a98f7e8, entries=150, sequenceid=416, filesize=12.0 K 2024-12-03T04:44:50,838 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 8eeca2704febb93c298048b6df24ffb0 in 967ms, sequenceid=416, compaction requested=false 2024-12-03T04:44:50,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:50,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:50,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-03T04:44:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-03T04:44:50,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-03T04:44:50,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2040 sec 2024-12-03T04:44:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:50,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T04:44:50,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:50,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:50,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:50,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:50,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:50,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:50,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.2140 sec 2024-12-03T04:44:50,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203062667cefc214d6fac67e68ceedde087_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201090383/Put/seqid=0 2024-12-03T04:44:50,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742235_1411 (size=14994) 2024-12-03T04:44:50,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201150927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:51,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201151022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:51,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201151041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:51,266 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:51,270 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203062667cefc214d6fac67e68ceedde087_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203062667cefc214d6fac67e68ceedde087_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:51,271 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/483477aa70d54651b5c9c43694b1dbe6, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:51,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/483477aa70d54651b5c9c43694b1dbe6 is 175, key is test_row_0/A:col10/1733201090383/Put/seqid=0 2024-12-03T04:44:51,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201151262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:51,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742236_1412 (size=39949) 2024-12-03T04:44:51,275 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=456, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/483477aa70d54651b5c9c43694b1dbe6 2024-12-03T04:44:51,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/767b1c8444b54126921c31ef99d93c78 is 50, key is test_row_0/B:col10/1733201090383/Put/seqid=0 2024-12-03T04:44:51,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742237_1413 (size=12301) 2024-12-03T04:44:51,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201151531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:51,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201151581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:51,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/767b1c8444b54126921c31ef99d93c78 2024-12-03T04:44:51,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/34251188545b437e8ef50ddfed82ac53 is 50, key is test_row_0/C:col10/1733201090383/Put/seqid=0 2024-12-03T04:44:51,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742238_1414 (size=12301) 2024-12-03T04:44:52,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/34251188545b437e8ef50ddfed82ac53 2024-12-03T04:44:52,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/483477aa70d54651b5c9c43694b1dbe6 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/483477aa70d54651b5c9c43694b1dbe6 2024-12-03T04:44:52,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/483477aa70d54651b5c9c43694b1dbe6, entries=200, sequenceid=456, filesize=39.0 K 2024-12-03T04:44:52,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201152101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/767b1c8444b54126921c31ef99d93c78 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/767b1c8444b54126921c31ef99d93c78 2024-12-03T04:44:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:52,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/767b1c8444b54126921c31ef99d93c78, entries=150, sequenceid=456, filesize=12.0 K 2024-12-03T04:44:52,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/34251188545b437e8ef50ddfed82ac53 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/34251188545b437e8ef50ddfed82ac53 2024-12-03T04:44:52,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/34251188545b437e8ef50ddfed82ac53, entries=150, sequenceid=456, filesize=12.0 K 2024-12-03T04:44:52,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 8eeca2704febb93c298048b6df24ffb0 in 1270ms, sequenceid=456, compaction requested=true 2024-12-03T04:44:52,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:52,116 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:52,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:52,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:52,117 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:52,117 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:52,117 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/A is initiating minor compaction (all files) 2024-12-03T04:44:52,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:52,117 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/A in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:52,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:52,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:52,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:52,117 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e66185cb9d974f90ba45359e6afac7d0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/06d94bb02d3f4782ad08b3fa53147c9a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/483477aa70d54651b5c9c43694b1dbe6] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=109.2 K 2024-12-03T04:44:52,117 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:52,117 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e66185cb9d974f90ba45359e6afac7d0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/06d94bb02d3f4782ad08b3fa53147c9a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/483477aa70d54651b5c9c43694b1dbe6] 2024-12-03T04:44:52,118 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting e66185cb9d974f90ba45359e6afac7d0, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733201088477 2024-12-03T04:44:52,118 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:52,118 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06d94bb02d3f4782ad08b3fa53147c9a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733201089845 2024-12-03T04:44:52,118 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/B is initiating minor compaction (all files) 2024-12-03T04:44:52,118 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/B in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:52,118 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/4215a661eec949c0bcea0d94eefd6004, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6bd237dc774ba1b2ef6c4a3b8fc2be, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/767b1c8444b54126921c31ef99d93c78] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=36.7 K 2024-12-03T04:44:52,119 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 483477aa70d54651b5c9c43694b1dbe6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733201090235 2024-12-03T04:44:52,119 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 4215a661eec949c0bcea0d94eefd6004, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733201088477 2024-12-03T04:44:52,120 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ff6bd237dc774ba1b2ef6c4a3b8fc2be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733201089845 2024-12-03T04:44:52,121 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 767b1c8444b54126921c31ef99d93c78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733201090351 2024-12-03T04:44:52,126 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:52,136 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:52,136 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/06858445ef004182bf1a701fb9672d9a is 50, key is test_row_0/B:col10/1733201090383/Put/seqid=0 2024-12-03T04:44:52,146 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412031dfe75f6336545b59cc2cbfd119e8dc2_8eeca2704febb93c298048b6df24ffb0 store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:52,147 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412031dfe75f6336545b59cc2cbfd119e8dc2_8eeca2704febb93c298048b6df24ffb0, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:52,147 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031dfe75f6336545b59cc2cbfd119e8dc2_8eeca2704febb93c298048b6df24ffb0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:52,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742239_1415 (size=13051) 2024-12-03T04:44:52,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742240_1416 (size=4469) 2024-12-03T04:44:52,171 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#A#compaction#347 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:52,171 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/156ea819f9aa4bd999863f86ffa3229d is 175, key is test_row_0/A:col10/1733201090383/Put/seqid=0 2024-12-03T04:44:52,172 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/06858445ef004182bf1a701fb9672d9a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/06858445ef004182bf1a701fb9672d9a 2024-12-03T04:44:52,181 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/B of 8eeca2704febb93c298048b6df24ffb0 into 06858445ef004182bf1a701fb9672d9a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:52,181 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:52,181 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/B, priority=13, startTime=1733201092116; duration=0sec 2024-12-03T04:44:52,181 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:52,181 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:B 2024-12-03T04:44:52,181 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:52,185 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:52,185 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/C is initiating minor compaction (all files) 2024-12-03T04:44:52,185 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/C in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:52,185 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/7ce87d96421f428d93c23daa612d065e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/e777036201b5435a91eb26493a98f7e8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/34251188545b437e8ef50ddfed82ac53] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=36.7 K 2024-12-03T04:44:52,189 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ce87d96421f428d93c23daa612d065e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733201088477 2024-12-03T04:44:52,190 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e777036201b5435a91eb26493a98f7e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733201089845 2024-12-03T04:44:52,190 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 34251188545b437e8ef50ddfed82ac53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733201090351 2024-12-03T04:44:52,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742241_1417 (size=32005) 2024-12-03T04:44:52,197 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#C#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:52,197 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/a0e7726db93b4081a82b66f6822aa3eb is 50, key is test_row_0/C:col10/1733201090383/Put/seqid=0 2024-12-03T04:44:52,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742242_1418 (size=13051) 2024-12-03T04:44:52,221 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/a0e7726db93b4081a82b66f6822aa3eb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/a0e7726db93b4081a82b66f6822aa3eb 2024-12-03T04:44:52,225 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/C of 8eeca2704febb93c298048b6df24ffb0 into a0e7726db93b4081a82b66f6822aa3eb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:52,225 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:52,225 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/C, priority=13, startTime=1733201092117; duration=0sec 2024-12-03T04:44:52,225 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:52,226 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:C 2024-12-03T04:44:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:52,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:44:52,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:52,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:52,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:52,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:52,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:52,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:52,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a74ffffa37e24437b85b878e9c156419_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201092555/Put/seqid=0 2024-12-03T04:44:52,602 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/156ea819f9aa4bd999863f86ffa3229d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/156ea819f9aa4bd999863f86ffa3229d 2024-12-03T04:44:52,606 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/A of 8eeca2704febb93c298048b6df24ffb0 into 156ea819f9aa4bd999863f86ffa3229d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:52,606 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:52,606 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/A, priority=13, startTime=1733201092116; duration=0sec 2024-12-03T04:44:52,606 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:52,606 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:A 2024-12-03T04:44:52,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742243_1419 (size=14994) 2024-12-03T04:44:52,619 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:52,623 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a74ffffa37e24437b85b878e9c156419_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a74ffffa37e24437b85b878e9c156419_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:52,624 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/0aa7bb5c638d49bc9b5e5bb19e5be9bb, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:52,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/0aa7bb5c638d49bc9b5e5bb19e5be9bb is 175, key is test_row_0/A:col10/1733201092555/Put/seqid=0 2024-12-03T04:44:52,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742244_1420 (size=39949) 2024-12-03T04:44:52,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201092702, max timestamp=1733201092766Num uniques 859; 2024-12-03T04:44:52,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:44:52,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201092702, max timestamp=1733201092766Num uniques 859; 2024-12-03T04:44:52,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:44:52,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733201092702, max timestamp=1733201092766Num uniques 859; 2024-12-03T04:44:52,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T04:44:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T04:44:52,802 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:52,802 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:44:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-03T04:44:52,804 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:44:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T04:44:52,805 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:44:52,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:44:52,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201152836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:52,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201152837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:52,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60776 deadline: 1733201152841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:52,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201152842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:52,852 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in 8eeca2704febb93c298048b6df24ffb0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:52,869 INFO [Thread-1510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1510. 2024-12-03T04:44:52,869 DEBUG [Thread-1510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:52,869 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:52,869 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:52,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:52,875 INFO [Thread-1502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1502. 2024-12-03T04:44:52,875 DEBUG [Thread-1502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:52,875 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:52,876 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:52,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:52,879 INFO [Thread-1508 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1508. 2024-12-03T04:44:52,879 DEBUG [Thread-1508 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:52,879 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:52,879 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:52,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:52,884 INFO [Thread-1506 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1506. 2024-12-03T04:44:52,884 DEBUG [Thread-1506 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:52,884 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:52,884 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:52,884 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:52,889 INFO [Thread-1504 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1504. 2024-12-03T04:44:52,889 DEBUG [Thread-1504 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:52,889 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:52,889 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:52,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T04:44:52,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60716 deadline: 1733201152952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:52,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T04:44:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:52,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:52,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733201152961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:52,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60746 deadline: 1733201152961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:53,056 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=468, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/0aa7bb5c638d49bc9b5e5bb19e5be9bb 2024-12-03T04:44:53,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/464406c34dd641c8a9f44baff4292a67 is 50, key is test_row_0/B:col10/1733201092555/Put/seqid=0 2024-12-03T04:44:53,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742245_1421 (size=12301) 2024-12-03T04:44:53,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/464406c34dd641c8a9f44baff4292a67 2024-12-03T04:44:53,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/4348dbbb2f8943128085b58571ed8713 is 50, key is test_row_0/C:col10/1733201092555/Put/seqid=0 2024-12-03T04:44:53,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742246_1422 (size=12301) 2024-12-03T04:44:53,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/4348dbbb2f8943128085b58571ed8713 2024-12-03T04:44:53,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/0aa7bb5c638d49bc9b5e5bb19e5be9bb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/0aa7bb5c638d49bc9b5e5bb19e5be9bb 2024-12-03T04:44:53,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/0aa7bb5c638d49bc9b5e5bb19e5be9bb, entries=200, sequenceid=468, filesize=39.0 K 2024-12-03T04:44:53,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/464406c34dd641c8a9f44baff4292a67 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/464406c34dd641c8a9f44baff4292a67 2024-12-03T04:44:53,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T04:44:53,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:53,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/464406c34dd641c8a9f44baff4292a67, entries=150, sequenceid=468, filesize=12.0 K 2024-12-03T04:44:53,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/4348dbbb2f8943128085b58571ed8713 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/4348dbbb2f8943128085b58571ed8713 2024-12-03T04:44:53,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:44:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60698 deadline: 1733201153122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 2024-12-03T04:44:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T04:44:53,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/4348dbbb2f8943128085b58571ed8713, entries=150, sequenceid=468, filesize=12.0 K 2024-12-03T04:44:53,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8eeca2704febb93c298048b6df24ffb0 in 565ms, sequenceid=468, compaction requested=false 2024-12-03T04:44:53,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:53,125 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in 8eeca2704febb93c298048b6df24ffb0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:44:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:44:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:44:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:53,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T04:44:53,171 INFO [Thread-1499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1499. 2024-12-03T04:44:53,171 DEBUG [Thread-1499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:53,171 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:53,171 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:53,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:44:53,177 INFO [Thread-1491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1491. 2024-12-03T04:44:53,177 DEBUG [Thread-1491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:53,177 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:53,178 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:53,178 INFO [Thread-1497 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1497. 2024-12-03T04:44:53,178 DEBUG [Thread-1497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:53,178 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:53,178 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:53,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:53,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:53,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ddc4a993288a456082eeea442a6fc6f5_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201092835/Put/seqid=0 2024-12-03T04:44:53,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742247_1423 (size=12454) 2024-12-03T04:44:53,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T04:44:53,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:53,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T04:44:53,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:53,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T04:44:53,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T04:44:53,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:53,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,588 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:53,592 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ddc4a993288a456082eeea442a6fc6f5_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ddc4a993288a456082eeea442a6fc6f5_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:53,593 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/7e056c1c65ba409abb28e4221a28d672, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:53,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/7e056c1c65ba409abb28e4221a28d672 is 175, key is test_row_0/A:col10/1733201092835/Put/seqid=0 2024-12-03T04:44:53,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742248_1424 (size=31255) 2024-12-03T04:44:53,601 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=506, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/7e056c1c65ba409abb28e4221a28d672 2024-12-03T04:44:53,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/e7c77b1dc24c4a06b92647b800c878bd is 50, key is test_row_0/B:col10/1733201092835/Put/seqid=0 2024-12-03T04:44:53,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742249_1425 (size=12301) 2024-12-03T04:44:53,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T04:44:53,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:53,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T04:44:53,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:53,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:53,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T04:44:54,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/e7c77b1dc24c4a06b92647b800c878bd 2024-12-03T04:44:54,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/323f5f094b0c47aa8d5efd7afb8d997b is 50, key is test_row_0/C:col10/1733201092835/Put/seqid=0 2024-12-03T04:44:54,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742250_1426 (size=12301) 2024-12-03T04:44:54,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/323f5f094b0c47aa8d5efd7afb8d997b 2024-12-03T04:44:54,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/7e056c1c65ba409abb28e4221a28d672 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7e056c1c65ba409abb28e4221a28d672 2024-12-03T04:44:54,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T04:44:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. as already flushing 2024-12-03T04:44:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:54,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:44:54,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7e056c1c65ba409abb28e4221a28d672, entries=150, sequenceid=506, filesize=30.5 K 2024-12-03T04:44:54,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/e7c77b1dc24c4a06b92647b800c878bd as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e7c77b1dc24c4a06b92647b800c878bd 2024-12-03T04:44:54,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e7c77b1dc24c4a06b92647b800c878bd, entries=150, sequenceid=506, filesize=12.0 K 2024-12-03T04:44:54,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/323f5f094b0c47aa8d5efd7afb8d997b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/323f5f094b0c47aa8d5efd7afb8d997b 2024-12-03T04:44:54,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/323f5f094b0c47aa8d5efd7afb8d997b, entries=150, sequenceid=506, filesize=12.0 K 2024-12-03T04:44:54,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=13.42 KB/13740 for 8eeca2704febb93c298048b6df24ffb0 in 884ms, sequenceid=506, compaction requested=true 2024-12-03T04:44:54,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:54,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:44:54,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:54,055 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:54,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:44:54,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:54,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eeca2704febb93c298048b6df24ffb0:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:44:54,055 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:54,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:54,056 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:54,056 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:54,056 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/B is initiating minor compaction (all files) 2024-12-03T04:44:54,056 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/A is initiating minor compaction (all files) 2024-12-03T04:44:54,056 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/A in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:54,056 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/B in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:54,056 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/06858445ef004182bf1a701fb9672d9a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/464406c34dd641c8a9f44baff4292a67, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e7c77b1dc24c4a06b92647b800c878bd] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=36.8 K 2024-12-03T04:44:54,056 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/156ea819f9aa4bd999863f86ffa3229d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/0aa7bb5c638d49bc9b5e5bb19e5be9bb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7e056c1c65ba409abb28e4221a28d672] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=100.8 K 2024-12-03T04:44:54,056 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:54,056 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/156ea819f9aa4bd999863f86ffa3229d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/0aa7bb5c638d49bc9b5e5bb19e5be9bb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7e056c1c65ba409abb28e4221a28d672] 2024-12-03T04:44:54,056 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 06858445ef004182bf1a701fb9672d9a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733201090351 2024-12-03T04:44:54,056 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 156ea819f9aa4bd999863f86ffa3229d, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733201090351 2024-12-03T04:44:54,057 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aa7bb5c638d49bc9b5e5bb19e5be9bb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733201090882 2024-12-03T04:44:54,057 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 464406c34dd641c8a9f44baff4292a67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733201090882 2024-12-03T04:44:54,057 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e056c1c65ba409abb28e4221a28d672, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1733201092834 2024-12-03T04:44:54,057 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e7c77b1dc24c4a06b92647b800c878bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1733201092834 2024-12-03T04:44:54,062 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:54,063 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#B#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:54,064 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412034620b6fec5cb4084a70d5bb8b28f088f_8eeca2704febb93c298048b6df24ffb0 store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:54,064 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/75e1567ddd924f60a46acb6d2de0a70c is 50, key is test_row_0/B:col10/1733201092835/Put/seqid=0 2024-12-03T04:44:54,067 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412034620b6fec5cb4084a70d5bb8b28f088f_8eeca2704febb93c298048b6df24ffb0, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:54,068 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034620b6fec5cb4084a70d5bb8b28f088f_8eeca2704febb93c298048b6df24ffb0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:54,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742251_1427 (size=13153) 2024-12-03T04:44:54,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742252_1428 (size=4469) 2024-12-03T04:44:54,074 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#A#compaction#356 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:54,074 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/4c7e2edbff4947a097e099c776ea1b0c is 175, key is test_row_0/A:col10/1733201092835/Put/seqid=0 2024-12-03T04:44:54,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742253_1429 (size=32107) 2024-12-03T04:44:54,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T04:44:54,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:54,193 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-03T04:44:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:54,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032411fc49a3ca4b03afacb44824954f1d_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201093173/Put/seqid=0 2024-12-03T04:44:54,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742254_1430 (size=9914) 2024-12-03T04:44:54,472 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/75e1567ddd924f60a46acb6d2de0a70c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/75e1567ddd924f60a46acb6d2de0a70c 2024-12-03T04:44:54,476 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/B of 8eeca2704febb93c298048b6df24ffb0 into 75e1567ddd924f60a46acb6d2de0a70c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:54,476 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:54,476 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/B, priority=13, startTime=1733201094055; duration=0sec 2024-12-03T04:44:54,476 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:44:54,476 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:B 2024-12-03T04:44:54,476 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:44:54,477 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:44:54,477 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 8eeca2704febb93c298048b6df24ffb0/C is initiating minor compaction (all files) 2024-12-03T04:44:54,477 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eeca2704febb93c298048b6df24ffb0/C in TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:54,477 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/a0e7726db93b4081a82b66f6822aa3eb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/4348dbbb2f8943128085b58571ed8713, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/323f5f094b0c47aa8d5efd7afb8d997b] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp, totalSize=36.8 K 2024-12-03T04:44:54,478 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a0e7726db93b4081a82b66f6822aa3eb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733201090351 2024-12-03T04:44:54,478 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 4348dbbb2f8943128085b58571ed8713, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733201090882 2024-12-03T04:44:54,478 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 323f5f094b0c47aa8d5efd7afb8d997b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1733201092834 2024-12-03T04:44:54,481 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/4c7e2edbff4947a097e099c776ea1b0c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/4c7e2edbff4947a097e099c776ea1b0c 2024-12-03T04:44:54,486 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/A of 8eeca2704febb93c298048b6df24ffb0 into 4c7e2edbff4947a097e099c776ea1b0c(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:54,486 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:54,486 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/A, priority=13, startTime=1733201094055; duration=0sec 2024-12-03T04:44:54,486 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:54,486 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:A 2024-12-03T04:44:54,486 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eeca2704febb93c298048b6df24ffb0#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:44:54,487 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/1b2d4c90a0754c81b60f375588ac7f27 is 50, key is test_row_0/C:col10/1733201092835/Put/seqid=0 2024-12-03T04:44:54,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742255_1431 (size=13153) 2024-12-03T04:44:54,498 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/1b2d4c90a0754c81b60f375588ac7f27 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/1b2d4c90a0754c81b60f375588ac7f27 2024-12-03T04:44:54,502 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eeca2704febb93c298048b6df24ffb0/C of 8eeca2704febb93c298048b6df24ffb0 into 1b2d4c90a0754c81b60f375588ac7f27(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:44:54,502 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:54,502 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., storeName=8eeca2704febb93c298048b6df24ffb0/C, priority=13, startTime=1733201094055; duration=0sec 2024-12-03T04:44:54,502 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:44:54,502 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eeca2704febb93c298048b6df24ffb0:C 2024-12-03T04:44:54,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:54,605 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032411fc49a3ca4b03afacb44824954f1d_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032411fc49a3ca4b03afacb44824954f1d_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:54,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/f21ad94660bd4a0e9601ecdfa13da44b, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:54,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/f21ad94660bd4a0e9601ecdfa13da44b is 175, key is test_row_0/A:col10/1733201093173/Put/seqid=0 2024-12-03T04:44:54,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742256_1432 (size=22561) 2024-12-03T04:44:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T04:44:55,010 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=511, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/f21ad94660bd4a0e9601ecdfa13da44b 2024-12-03T04:44:55,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ac192b05557947bc82b38e77fd3a4ebd is 50, key is test_row_0/B:col10/1733201093173/Put/seqid=0 2024-12-03T04:44:55,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742257_1433 (size=9857) 2024-12-03T04:44:55,153 INFO [Thread-1493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1493. 2024-12-03T04:44:55,153 DEBUG [Thread-1493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:55,153 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:55,154 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:55,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:55,420 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ac192b05557947bc82b38e77fd3a4ebd 2024-12-03T04:44:55,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/c61ddaac78684245b7c011a17e9c3bba is 50, key is test_row_0/C:col10/1733201093173/Put/seqid=0 2024-12-03T04:44:55,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742258_1434 (size=9857) 2024-12-03T04:44:55,828 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/c61ddaac78684245b7c011a17e9c3bba 2024-12-03T04:44:55,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/f21ad94660bd4a0e9601ecdfa13da44b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/f21ad94660bd4a0e9601ecdfa13da44b 2024-12-03T04:44:55,834 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/f21ad94660bd4a0e9601ecdfa13da44b, entries=100, sequenceid=511, filesize=22.0 K 2024-12-03T04:44:55,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/ac192b05557947bc82b38e77fd3a4ebd as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ac192b05557947bc82b38e77fd3a4ebd 2024-12-03T04:44:55,838 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ac192b05557947bc82b38e77fd3a4ebd, entries=100, sequenceid=511, filesize=9.6 K 2024-12-03T04:44:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/c61ddaac78684245b7c011a17e9c3bba as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/c61ddaac78684245b7c011a17e9c3bba 2024-12-03T04:44:55,841 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/c61ddaac78684245b7c011a17e9c3bba, entries=100, sequenceid=511, filesize=9.6 K 2024-12-03T04:44:55,842 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=6.71 KB/6870 for 8eeca2704febb93c298048b6df24ffb0 in 1649ms, sequenceid=511, compaction requested=false 2024-12-03T04:44:55,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 8eeca2704febb93c298048b6df24ffb0: 2024-12-03T04:44:55,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:55,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-03T04:44:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-03T04:44:55,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-03T04:44:55,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0380 sec 2024-12-03T04:44:55,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.0420 sec 2024-12-03T04:44:56,873 INFO [Thread-1495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1495. 2024-12-03T04:44:56,873 DEBUG [Thread-1495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:44:56,873 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:56,873 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:56,873 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T04:44:56,962 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 164 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 842 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2522 rows 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 853 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2554 rows 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 850 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2549 rows 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 847 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2541 rows 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 848 2024-12-03T04:44:56,962 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2541 rows 2024-12-03T04:44:56,963 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T04:44:56,963 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T04:44:56,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:56,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:44:56,963 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:44:56,964 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T04:44:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T04:44:56,966 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201096966"}]},"ts":"1733201096966"} 2024-12-03T04:44:56,967 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T04:44:56,967 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T04:44:56,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T04:44:56,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, UNASSIGN}] 2024-12-03T04:44:56,970 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, UNASSIGN 2024-12-03T04:44:56,970 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=8eeca2704febb93c298048b6df24ffb0, regionState=CLOSING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:44:56,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, UNASSIGN because future has completed 2024-12-03T04:44:56,972 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T04:44:56,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:44:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T04:44:57,123 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:57,123 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T04:44:57,123 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing 8eeca2704febb93c298048b6df24ffb0, disabling compactions & flushes 2024-12-03T04:44:57,123 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:57,123 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:57,123 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. after waiting 0 ms 2024-12-03T04:44:57,123 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:57,124 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing 8eeca2704febb93c298048b6df24ffb0 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-03T04:44:57,124 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=A 2024-12-03T04:44:57,124 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:57,124 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=B 2024-12-03T04:44:57,124 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:57,124 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8eeca2704febb93c298048b6df24ffb0, store=C 2024-12-03T04:44:57,124 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:44:57,128 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203035aa9bb88bd41b9b8f6a3ccc2b1e6cc_8eeca2704febb93c298048b6df24ffb0 is 50, key is test_row_0/A:col10/1733201095152/Put/seqid=0 2024-12-03T04:44:57,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742259_1435 (size=9914) 2024-12-03T04:44:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T04:44:57,532 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:57,536 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203035aa9bb88bd41b9b8f6a3ccc2b1e6cc_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203035aa9bb88bd41b9b8f6a3ccc2b1e6cc_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:57,537 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/fd845c80c66e46ec9c05891f51249c6d, store: [table=TestAcidGuarantees family=A region=8eeca2704febb93c298048b6df24ffb0] 2024-12-03T04:44:57,537 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/fd845c80c66e46ec9c05891f51249c6d is 175, key is test_row_0/A:col10/1733201095152/Put/seqid=0 2024-12-03T04:44:57,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742260_1436 (size=22561) 2024-12-03T04:44:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T04:44:57,941 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=519, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/fd845c80c66e46ec9c05891f51249c6d 2024-12-03T04:44:57,949 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/1c91de47f60147fe800a2aac0c5a3d8e is 50, key is test_row_0/B:col10/1733201095152/Put/seqid=0 2024-12-03T04:44:57,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742261_1437 (size=9857) 2024-12-03T04:44:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T04:44:58,358 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/1c91de47f60147fe800a2aac0c5a3d8e 2024-12-03T04:44:58,365 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/5164e9398cd5480c8ba33bb47f8e0464 is 50, key is test_row_0/C:col10/1733201095152/Put/seqid=0 2024-12-03T04:44:58,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742262_1438 (size=9857) 2024-12-03T04:44:58,769 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/5164e9398cd5480c8ba33bb47f8e0464 2024-12-03T04:44:58,772 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/A/fd845c80c66e46ec9c05891f51249c6d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/fd845c80c66e46ec9c05891f51249c6d 2024-12-03T04:44:58,775 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/fd845c80c66e46ec9c05891f51249c6d, entries=100, sequenceid=519, filesize=22.0 K 2024-12-03T04:44:58,776 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/B/1c91de47f60147fe800a2aac0c5a3d8e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/1c91de47f60147fe800a2aac0c5a3d8e 2024-12-03T04:44:58,778 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/1c91de47f60147fe800a2aac0c5a3d8e, entries=100, sequenceid=519, filesize=9.6 K 2024-12-03T04:44:58,779 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/.tmp/C/5164e9398cd5480c8ba33bb47f8e0464 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/5164e9398cd5480c8ba33bb47f8e0464 2024-12-03T04:44:58,781 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/5164e9398cd5480c8ba33bb47f8e0464, entries=100, sequenceid=519, filesize=9.6 K 2024-12-03T04:44:58,782 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 8eeca2704febb93c298048b6df24ffb0 in 1659ms, sequenceid=519, compaction requested=true 2024-12-03T04:44:58,782 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/cad7ef2ad753457d977af59ce1351658, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/005becf953584b75a73a789770618a04, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3afbcc9aef284929a6f479331d812485, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/db374f42bb3a4d53bc7810483c9c4498, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a682417305b44ef49d4e76d0320a0ace, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3b7920fc9b924089ba3e3281af6b52fb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e1cbc6c239c840959cc5a5a9a8e5d8cb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ce1aeaad9db342d5a080d11b4a96bf1d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ffde27de0bad46e385a26ed05932bfbd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/2851515ff1c14bd9b9be9e5df40ea498, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/07e4b2382ccd4257889cec756eca842f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5d3e6f34988341c4b57e7f58e59a8666, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/d0e1a8f4df304138b41c802e262f584c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/44d476075aa94ec99cb5fa8f74e7728f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7d12d9186aad4b7baade3627a2a32c13, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5e19ffd0cb5a46b9ba50379bcb77cd82, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a14b5ab5ae1648a8b9f13dcb5062392c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e51eaaebaab5465fa392a930c9df12a2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/29b65ea1ac0141b0b995dc2a5794f5ff, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e66185cb9d974f90ba45359e6afac7d0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/06d94bb02d3f4782ad08b3fa53147c9a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/483477aa70d54651b5c9c43694b1dbe6, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/156ea819f9aa4bd999863f86ffa3229d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/0aa7bb5c638d49bc9b5e5bb19e5be9bb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7e056c1c65ba409abb28e4221a28d672] to archive 2024-12-03T04:44:58,783 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:44:58,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/cad7ef2ad753457d977af59ce1351658 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/cad7ef2ad753457d977af59ce1351658 2024-12-03T04:44:58,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/005becf953584b75a73a789770618a04 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/005becf953584b75a73a789770618a04 2024-12-03T04:44:58,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3afbcc9aef284929a6f479331d812485 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3afbcc9aef284929a6f479331d812485 2024-12-03T04:44:58,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/db374f42bb3a4d53bc7810483c9c4498 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/db374f42bb3a4d53bc7810483c9c4498 2024-12-03T04:44:58,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a682417305b44ef49d4e76d0320a0ace to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a682417305b44ef49d4e76d0320a0ace 2024-12-03T04:44:58,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3b7920fc9b924089ba3e3281af6b52fb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/3b7920fc9b924089ba3e3281af6b52fb 2024-12-03T04:44:58,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e1cbc6c239c840959cc5a5a9a8e5d8cb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e1cbc6c239c840959cc5a5a9a8e5d8cb 2024-12-03T04:44:58,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ce1aeaad9db342d5a080d11b4a96bf1d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ce1aeaad9db342d5a080d11b4a96bf1d 2024-12-03T04:44:58,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ffde27de0bad46e385a26ed05932bfbd to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/ffde27de0bad46e385a26ed05932bfbd 2024-12-03T04:44:58,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/2851515ff1c14bd9b9be9e5df40ea498 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/2851515ff1c14bd9b9be9e5df40ea498 2024-12-03T04:44:58,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/07e4b2382ccd4257889cec756eca842f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/07e4b2382ccd4257889cec756eca842f 2024-12-03T04:44:58,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5d3e6f34988341c4b57e7f58e59a8666 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5d3e6f34988341c4b57e7f58e59a8666 2024-12-03T04:44:58,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/d0e1a8f4df304138b41c802e262f584c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/d0e1a8f4df304138b41c802e262f584c 2024-12-03T04:44:58,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/44d476075aa94ec99cb5fa8f74e7728f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/44d476075aa94ec99cb5fa8f74e7728f 2024-12-03T04:44:58,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7d12d9186aad4b7baade3627a2a32c13 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7d12d9186aad4b7baade3627a2a32c13 2024-12-03T04:44:58,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5e19ffd0cb5a46b9ba50379bcb77cd82 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/5e19ffd0cb5a46b9ba50379bcb77cd82 2024-12-03T04:44:58,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a14b5ab5ae1648a8b9f13dcb5062392c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/a14b5ab5ae1648a8b9f13dcb5062392c 2024-12-03T04:44:58,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e51eaaebaab5465fa392a930c9df12a2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e51eaaebaab5465fa392a930c9df12a2 2024-12-03T04:44:58,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/29b65ea1ac0141b0b995dc2a5794f5ff to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/29b65ea1ac0141b0b995dc2a5794f5ff 2024-12-03T04:44:58,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e66185cb9d974f90ba45359e6afac7d0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/e66185cb9d974f90ba45359e6afac7d0 2024-12-03T04:44:58,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/06d94bb02d3f4782ad08b3fa53147c9a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/06d94bb02d3f4782ad08b3fa53147c9a 2024-12-03T04:44:58,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/483477aa70d54651b5c9c43694b1dbe6 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/483477aa70d54651b5c9c43694b1dbe6 2024-12-03T04:44:58,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/156ea819f9aa4bd999863f86ffa3229d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/156ea819f9aa4bd999863f86ffa3229d 2024-12-03T04:44:58,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/0aa7bb5c638d49bc9b5e5bb19e5be9bb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/0aa7bb5c638d49bc9b5e5bb19e5be9bb 2024-12-03T04:44:58,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7e056c1c65ba409abb28e4221a28d672 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/7e056c1c65ba409abb28e4221a28d672 2024-12-03T04:44:58,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/d300f6d1bc4d467186c290b6a16bcdd5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/c8cd56e13071487885c90a89f09bc341, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ef511c1d9dd74441b0b4ed90154bb929, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/21b27757210648e3b9b7138d5c2581b5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/3eeae650995446098741e3b0e0e73a26, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6fdbce45714ef59938933bd45b2b4d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/33c3e86f737748d287ebf15d7accf180, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/486e6016174743668b88716f31828e3f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/44020fba902f45008220f6033e2fc172, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/4503d902947e4afea3d1ade7420d061e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e2795ba6207345fa9e8bfe931739d8b0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/161fe4da64d34520b3c3537488f84822, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/a9060c9dbf7e43e5b2bc17231ba54c82, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/85ef6a1c396646a88b3111d0273df30d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/48d6e12038574e159b231e7ba16fdb30, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/398cb2be19ac44ada83da63e7906e2b8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/bf9eaa97c3f34818b27a10b2c0d27816, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ae38edcfdb03494a95ce659b6c220196, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/4215a661eec949c0bcea0d94eefd6004, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e3140715bd6b44d59fb55046ee2400b6, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6bd237dc774ba1b2ef6c4a3b8fc2be, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/06858445ef004182bf1a701fb9672d9a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/767b1c8444b54126921c31ef99d93c78, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/464406c34dd641c8a9f44baff4292a67, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e7c77b1dc24c4a06b92647b800c878bd] to archive 2024-12-03T04:44:58,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:44:58,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/d300f6d1bc4d467186c290b6a16bcdd5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/d300f6d1bc4d467186c290b6a16bcdd5 2024-12-03T04:44:58,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/c8cd56e13071487885c90a89f09bc341 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/c8cd56e13071487885c90a89f09bc341 2024-12-03T04:44:58,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ef511c1d9dd74441b0b4ed90154bb929 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ef511c1d9dd74441b0b4ed90154bb929 2024-12-03T04:44:58,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/21b27757210648e3b9b7138d5c2581b5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/21b27757210648e3b9b7138d5c2581b5 2024-12-03T04:44:58,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/3eeae650995446098741e3b0e0e73a26 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/3eeae650995446098741e3b0e0e73a26 2024-12-03T04:44:58,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6fdbce45714ef59938933bd45b2b4d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6fdbce45714ef59938933bd45b2b4d 2024-12-03T04:44:58,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/33c3e86f737748d287ebf15d7accf180 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/33c3e86f737748d287ebf15d7accf180 2024-12-03T04:44:58,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/486e6016174743668b88716f31828e3f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/486e6016174743668b88716f31828e3f 2024-12-03T04:44:58,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/44020fba902f45008220f6033e2fc172 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/44020fba902f45008220f6033e2fc172 2024-12-03T04:44:58,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/4503d902947e4afea3d1ade7420d061e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/4503d902947e4afea3d1ade7420d061e 2024-12-03T04:44:58,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e2795ba6207345fa9e8bfe931739d8b0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e2795ba6207345fa9e8bfe931739d8b0 2024-12-03T04:44:58,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/161fe4da64d34520b3c3537488f84822 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/161fe4da64d34520b3c3537488f84822 2024-12-03T04:44:58,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/a9060c9dbf7e43e5b2bc17231ba54c82 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/a9060c9dbf7e43e5b2bc17231ba54c82 2024-12-03T04:44:58,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/85ef6a1c396646a88b3111d0273df30d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/85ef6a1c396646a88b3111d0273df30d 2024-12-03T04:44:58,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/48d6e12038574e159b231e7ba16fdb30 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/48d6e12038574e159b231e7ba16fdb30 2024-12-03T04:44:58,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/398cb2be19ac44ada83da63e7906e2b8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/398cb2be19ac44ada83da63e7906e2b8 2024-12-03T04:44:58,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/bf9eaa97c3f34818b27a10b2c0d27816 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/bf9eaa97c3f34818b27a10b2c0d27816 2024-12-03T04:44:58,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ae38edcfdb03494a95ce659b6c220196 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ae38edcfdb03494a95ce659b6c220196 2024-12-03T04:44:58,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/4215a661eec949c0bcea0d94eefd6004 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/4215a661eec949c0bcea0d94eefd6004 2024-12-03T04:44:58,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e3140715bd6b44d59fb55046ee2400b6 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e3140715bd6b44d59fb55046ee2400b6 2024-12-03T04:44:58,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6bd237dc774ba1b2ef6c4a3b8fc2be to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ff6bd237dc774ba1b2ef6c4a3b8fc2be 2024-12-03T04:44:58,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/06858445ef004182bf1a701fb9672d9a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/06858445ef004182bf1a701fb9672d9a 2024-12-03T04:44:58,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/767b1c8444b54126921c31ef99d93c78 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/767b1c8444b54126921c31ef99d93c78 2024-12-03T04:44:58,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/464406c34dd641c8a9f44baff4292a67 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/464406c34dd641c8a9f44baff4292a67 2024-12-03T04:44:58,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e7c77b1dc24c4a06b92647b800c878bd to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/e7c77b1dc24c4a06b92647b800c878bd 2024-12-03T04:44:58,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/74d8dfbbd1774ff08b31b50cf8d2e2ad, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f3a64acb25fe4b4093fca25e72595c65, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/0458758eff104277bf9296a5edc14847, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/3bbab6d8113d40b4a12a7cfbf017958d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/98f807210abe461f995589fe1402c4ad, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/edb1edc719804b808d171c2bddf4c737, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/b4e4bdf6c05640f6966196cfaff7ef0d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/24e850005a6b4fd09322a6dd41c06513, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/d7d1a8c84ec9417caa49dfb8ed78cd8c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/645fe4a6bef849d5b80a18125e89866f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/899195064281416a90123222f33aa47e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/fa4a6c99347949499229680530ab20c8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/def02be04b28407281a3926a7285860e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/392f5f14e71a4da391cc9cbde25e25a7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/756962a7a1d64a09b27afe633085b9d9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/22b37c9b8ed1412fbf2f892d68c23d55, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f7cea33a540e42418f34c92dde079c10, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/81d80f1bd4304b5e83d0a5779d56d04c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/7ce87d96421f428d93c23daa612d065e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/b9483cdb4c45452bb1fe0f96dc0b78a8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/e777036201b5435a91eb26493a98f7e8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/a0e7726db93b4081a82b66f6822aa3eb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/34251188545b437e8ef50ddfed82ac53, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/4348dbbb2f8943128085b58571ed8713, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/323f5f094b0c47aa8d5efd7afb8d997b] to archive 2024-12-03T04:44:58,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:44:58,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/74d8dfbbd1774ff08b31b50cf8d2e2ad to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/74d8dfbbd1774ff08b31b50cf8d2e2ad 2024-12-03T04:44:58,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f3a64acb25fe4b4093fca25e72595c65 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f3a64acb25fe4b4093fca25e72595c65 2024-12-03T04:44:58,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/0458758eff104277bf9296a5edc14847 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/0458758eff104277bf9296a5edc14847 2024-12-03T04:44:58,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/3bbab6d8113d40b4a12a7cfbf017958d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/3bbab6d8113d40b4a12a7cfbf017958d 2024-12-03T04:44:58,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/98f807210abe461f995589fe1402c4ad to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/98f807210abe461f995589fe1402c4ad 2024-12-03T04:44:58,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/edb1edc719804b808d171c2bddf4c737 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/edb1edc719804b808d171c2bddf4c737 2024-12-03T04:44:58,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/b4e4bdf6c05640f6966196cfaff7ef0d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/b4e4bdf6c05640f6966196cfaff7ef0d 2024-12-03T04:44:58,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/24e850005a6b4fd09322a6dd41c06513 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/24e850005a6b4fd09322a6dd41c06513 2024-12-03T04:44:58,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/d7d1a8c84ec9417caa49dfb8ed78cd8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/d7d1a8c84ec9417caa49dfb8ed78cd8c 2024-12-03T04:44:58,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/645fe4a6bef849d5b80a18125e89866f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/645fe4a6bef849d5b80a18125e89866f 2024-12-03T04:44:58,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/899195064281416a90123222f33aa47e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/899195064281416a90123222f33aa47e 2024-12-03T04:44:58,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/fa4a6c99347949499229680530ab20c8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/fa4a6c99347949499229680530ab20c8 2024-12-03T04:44:58,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/def02be04b28407281a3926a7285860e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/def02be04b28407281a3926a7285860e 2024-12-03T04:44:58,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/392f5f14e71a4da391cc9cbde25e25a7 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/392f5f14e71a4da391cc9cbde25e25a7 2024-12-03T04:44:58,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/756962a7a1d64a09b27afe633085b9d9 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/756962a7a1d64a09b27afe633085b9d9 2024-12-03T04:44:58,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/22b37c9b8ed1412fbf2f892d68c23d55 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/22b37c9b8ed1412fbf2f892d68c23d55 2024-12-03T04:44:58,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f7cea33a540e42418f34c92dde079c10 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/f7cea33a540e42418f34c92dde079c10 2024-12-03T04:44:58,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/81d80f1bd4304b5e83d0a5779d56d04c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/81d80f1bd4304b5e83d0a5779d56d04c 2024-12-03T04:44:58,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/7ce87d96421f428d93c23daa612d065e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/7ce87d96421f428d93c23daa612d065e 2024-12-03T04:44:58,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/b9483cdb4c45452bb1fe0f96dc0b78a8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/b9483cdb4c45452bb1fe0f96dc0b78a8 2024-12-03T04:44:58,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/e777036201b5435a91eb26493a98f7e8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/e777036201b5435a91eb26493a98f7e8 2024-12-03T04:44:58,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/a0e7726db93b4081a82b66f6822aa3eb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/a0e7726db93b4081a82b66f6822aa3eb 2024-12-03T04:44:58,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/34251188545b437e8ef50ddfed82ac53 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/34251188545b437e8ef50ddfed82ac53 2024-12-03T04:44:58,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/4348dbbb2f8943128085b58571ed8713 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/4348dbbb2f8943128085b58571ed8713 2024-12-03T04:44:58,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/323f5f094b0c47aa8d5efd7afb8d997b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/323f5f094b0c47aa8d5efd7afb8d997b 2024-12-03T04:44:58,847 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/recovered.edits/522.seqid, newMaxSeqId=522, maxSeqId=4 2024-12-03T04:44:58,848 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. 2024-12-03T04:44:58,848 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for 8eeca2704febb93c298048b6df24ffb0: Waiting for close lock at 1733201097123Running coprocessor pre-close hooks at 1733201097123Disabling compacts and flushes for region at 1733201097123Disabling writes for close at 1733201097123Obtaining lock to block concurrent updates at 1733201097124 (+1 ms)Preparing flush snapshotting stores in 8eeca2704febb93c298048b6df24ffb0 at 1733201097124Finished memstore snapshotting TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733201097124Flushing stores of TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0. at 1733201097124Flushing 8eeca2704febb93c298048b6df24ffb0/A: creating writer at 1733201097124Flushing mob file 8eeca2704febb93c298048b6df24ffb0/A: appending metadata at 1733201097128 (+4 ms)Flushing mob file 8eeca2704febb93c298048b6df24ffb0/A: closing flushed file at 1733201097128Flushing 8eeca2704febb93c298048b6df24ffb0/A: appending metadata at 1733201097537 (+409 ms)Flushing 8eeca2704febb93c298048b6df24ffb0/A: closing flushed file at 1733201097537Flushing 8eeca2704febb93c298048b6df24ffb0/B: creating writer at 1733201097945 (+408 ms)Flushing 8eeca2704febb93c298048b6df24ffb0/B: appending metadata at 1733201097948 (+3 ms)Flushing 8eeca2704febb93c298048b6df24ffb0/B: closing flushed file at 1733201097948Flushing 8eeca2704febb93c298048b6df24ffb0/C: creating writer at 1733201098362 (+414 ms)Flushing 8eeca2704febb93c298048b6df24ffb0/C: appending metadata at 1733201098364 (+2 ms)Flushing 8eeca2704febb93c298048b6df24ffb0/C: closing flushed file at 1733201098364Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c56a15: reopening flushed file at 1733201098772 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@133a1e85: reopening flushed file at 1733201098775 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27c9d1c: reopening flushed file at 1733201098778 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 8eeca2704febb93c298048b6df24ffb0 in 1659ms, sequenceid=519, compaction requested=true at 1733201098782 (+4 ms)Writing region close event to WAL at 1733201098845 (+63 ms)Running coprocessor post-close hooks at 1733201098848 (+3 ms)Closed at 1733201098848 2024-12-03T04:44:58,849 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed 8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:58,850 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=8eeca2704febb93c298048b6df24ffb0, regionState=CLOSED 2024-12-03T04:44:58,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:44:58,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-03T04:44:58,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure 8eeca2704febb93c298048b6df24ffb0, server=32beccd76b35,43193,1733200986730 in 1.8800 sec 2024-12-03T04:44:58,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-12-03T04:44:58,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eeca2704febb93c298048b6df24ffb0, UNASSIGN in 1.8840 sec 2024-12-03T04:44:58,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-03T04:44:58,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8870 sec 2024-12-03T04:44:58,858 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201098858"}]},"ts":"1733201098858"} 2024-12-03T04:44:58,859 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T04:44:58,859 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T04:44:58,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8950 sec 2024-12-03T04:44:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T04:44:59,112 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:59,112 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T04:44:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:59,114 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T04:44:59,114 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:59,117 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,119 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/recovered.edits] 2024-12-03T04:44:59,121 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/4c7e2edbff4947a097e099c776ea1b0c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/4c7e2edbff4947a097e099c776ea1b0c 2024-12-03T04:44:59,122 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/f21ad94660bd4a0e9601ecdfa13da44b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/f21ad94660bd4a0e9601ecdfa13da44b 2024-12-03T04:44:59,123 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/fd845c80c66e46ec9c05891f51249c6d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/A/fd845c80c66e46ec9c05891f51249c6d 2024-12-03T04:44:59,125 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/1c91de47f60147fe800a2aac0c5a3d8e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/1c91de47f60147fe800a2aac0c5a3d8e 2024-12-03T04:44:59,126 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/75e1567ddd924f60a46acb6d2de0a70c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/75e1567ddd924f60a46acb6d2de0a70c 2024-12-03T04:44:59,127 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ac192b05557947bc82b38e77fd3a4ebd to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/B/ac192b05557947bc82b38e77fd3a4ebd 2024-12-03T04:44:59,128 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/1b2d4c90a0754c81b60f375588ac7f27 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/1b2d4c90a0754c81b60f375588ac7f27 2024-12-03T04:44:59,130 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/5164e9398cd5480c8ba33bb47f8e0464 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/5164e9398cd5480c8ba33bb47f8e0464 2024-12-03T04:44:59,130 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/c61ddaac78684245b7c011a17e9c3bba to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/C/c61ddaac78684245b7c011a17e9c3bba 2024-12-03T04:44:59,132 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/recovered.edits/522.seqid to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0/recovered.edits/522.seqid 2024-12-03T04:44:59,133 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,133 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T04:44:59,133 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T04:44:59,134 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T04:44:59,136 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203035aa9bb88bd41b9b8f6a3ccc2b1e6cc_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203035aa9bb88bd41b9b8f6a3ccc2b1e6cc_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,137 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203042260e132634c9c9fea2ba287ee8379_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203042260e132634c9c9fea2ba287ee8379_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,138 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203062667cefc214d6fac67e68ceedde087_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203062667cefc214d6fac67e68ceedde087_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,138 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030878e9fe0c3c496a8d4db52179880fdb_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030878e9fe0c3c496a8d4db52179880fdb_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,139 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203130c6a51384f4e8e9615b222d6bcf933_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203130c6a51384f4e8e9615b222d6bcf933_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,140 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032411fc49a3ca4b03afacb44824954f1d_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032411fc49a3ca4b03afacb44824954f1d_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,141 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036e98a9d617794e33b2b98ed1e5e1b350_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036e98a9d617794e33b2b98ed1e5e1b350_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,142 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389ef0a345f5f4805bbab44f9592306c8_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389ef0a345f5f4805bbab44f9592306c8_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,143 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039168efcf712840f8b1463f5c286e1646_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039168efcf712840f8b1463f5c286e1646_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,144 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a74ffffa37e24437b85b878e9c156419_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a74ffffa37e24437b85b878e9c156419_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,145 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b06945648042445e887ea5959c44199b_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b06945648042445e887ea5959c44199b_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,146 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b64c53a105de47429aa118fbc3dd17d1_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b64c53a105de47429aa118fbc3dd17d1_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,147 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203beee9b14bad94654b3c16994946f2010_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203beee9b14bad94654b3c16994946f2010_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,148 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ce29a56cd15244cd959673c217919118_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ce29a56cd15244cd959673c217919118_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,148 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d462a945a2ed4c77acd393f0a50d5834_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d462a945a2ed4c77acd393f0a50d5834_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,149 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d9e23aa130a04bdab9ad7908fead7f22_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d9e23aa130a04bdab9ad7908fead7f22_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,150 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ddc4a993288a456082eeea442a6fc6f5_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ddc4a993288a456082eeea442a6fc6f5_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,151 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203df15b08a69704b5a902c04fd762bc616_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203df15b08a69704b5a902c04fd762bc616_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,152 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e539970562f84ce8b450c8551f596827_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e539970562f84ce8b450c8551f596827_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,153 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fb4af61eb1244a2884134f8ac8843bd6_8eeca2704febb93c298048b6df24ffb0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fb4af61eb1244a2884134f8ac8843bd6_8eeca2704febb93c298048b6df24ffb0 2024-12-03T04:44:59,154 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T04:44:59,155 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:59,157 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T04:44:59,159 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T04:44:59,159 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:59,160 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T04:44:59,160 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733201099160"}]},"ts":"9223372036854775807"} 2024-12-03T04:44:59,161 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T04:44:59,161 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8eeca2704febb93c298048b6df24ffb0, NAME => 'TestAcidGuarantees,,1733201070742.8eeca2704febb93c298048b6df24ffb0.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T04:44:59,161 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T04:44:59,162 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733201099161"}]},"ts":"9223372036854775807"} 2024-12-03T04:44:59,163 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T04:44:59,163 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:59,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-12-03T04:44:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T04:44:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T04:44:59,221 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:44:59,233 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 250), OpenFileDescriptor=461 (was 459) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=838 (was 834) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3528 (was 4106) 2024-12-03T04:44:59,241 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=838, ProcessCount=11, AvailableMemoryMB=3528 2024-12-03T04:44:59,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T04:44:59,242 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:44:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T04:44:59,244 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T04:44:59,244 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:44:59,244 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-12-03T04:44:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-03T04:44:59,245 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T04:44:59,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742263_1439 (size=1061) 2024-12-03T04:44:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-03T04:44:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-03T04:44:59,651 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676 2024-12-03T04:44:59,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742264_1440 (size=53) 2024-12-03T04:44:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-03T04:45:00,056 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:45:00,056 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a9660218570dfb53504238e06a5637bb, disabling compactions & flushes 2024-12-03T04:45:00,056 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:00,056 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:00,056 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. after waiting 0 ms 2024-12-03T04:45:00,056 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:00,056 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:00,056 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a9660218570dfb53504238e06a5637bb: Waiting for close lock at 1733201100056Disabling compacts and flushes for region at 1733201100056Disabling writes for close at 1733201100056Writing region close event to WAL at 1733201100056Closed at 1733201100056 2024-12-03T04:45:00,057 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T04:45:00,058 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733201100058"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733201100058"}]},"ts":"1733201100058"} 2024-12-03T04:45:00,059 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T04:45:00,060 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T04:45:00,060 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201100060"}]},"ts":"1733201100060"} 2024-12-03T04:45:00,061 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T04:45:00,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9660218570dfb53504238e06a5637bb, ASSIGN}] 2024-12-03T04:45:00,062 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9660218570dfb53504238e06a5637bb, ASSIGN 2024-12-03T04:45:00,063 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9660218570dfb53504238e06a5637bb, ASSIGN; state=OFFLINE, location=32beccd76b35,43193,1733200986730; forceNewPlan=false, retain=false 2024-12-03T04:45:00,213 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=a9660218570dfb53504238e06a5637bb, regionState=OPENING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:45:00,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9660218570dfb53504238e06a5637bb, ASSIGN because future has completed 2024-12-03T04:45:00,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:45:00,369 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:00,369 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:45:00,370 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,370 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:45:00,370 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,370 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,372 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,373 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:45:00,373 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9660218570dfb53504238e06a5637bb columnFamilyName A 2024-12-03T04:45:00,373 DEBUG [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:00,373 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] regionserver.HStore(327): Store=a9660218570dfb53504238e06a5637bb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:45:00,373 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,374 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:45:00,374 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9660218570dfb53504238e06a5637bb columnFamilyName B 2024-12-03T04:45:00,374 DEBUG [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:00,375 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] regionserver.HStore(327): Store=a9660218570dfb53504238e06a5637bb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:45:00,375 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,375 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:45:00,376 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9660218570dfb53504238e06a5637bb columnFamilyName C 2024-12-03T04:45:00,376 DEBUG [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:00,376 INFO [StoreOpener-a9660218570dfb53504238e06a5637bb-1 {}] regionserver.HStore(327): Store=a9660218570dfb53504238e06a5637bb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:45:00,376 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:00,376 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,377 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,377 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,378 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,378 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,378 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:45:00,379 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,381 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T04:45:00,381 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened a9660218570dfb53504238e06a5637bb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72287021, jitterRate=0.0771605521440506}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:45:00,382 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-03T04:45:00,382 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for a9660218570dfb53504238e06a5637bb: Running coprocessor pre-open hook at 1733201100370Writing region info on filesystem at 1733201100370Initializing all the Stores at 1733201100371 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201100371Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201100371Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201100371Cleaning up temporary data from old regions at 1733201100378 (+7 ms)Running coprocessor post-open hooks at 1733201100382 (+4 ms)Region opened successfully at 1733201100382 2024-12-03T04:45:00,383 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., pid=108, masterSystemTime=1733201100367 2024-12-03T04:45:00,385 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:00,385 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:00,385 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=a9660218570dfb53504238e06a5637bb, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:45:00,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:45:00,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-03T04:45:00,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 in 172 msec 2024-12-03T04:45:00,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-03T04:45:00,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9660218570dfb53504238e06a5637bb, ASSIGN in 328 msec 2024-12-03T04:45:00,391 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T04:45:00,392 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201100391"}]},"ts":"1733201100391"} 2024-12-03T04:45:00,393 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T04:45:00,393 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T04:45:00,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-12-03T04:45:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-03T04:45:01,392 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:01,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:01,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:01,394 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3362a79f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:01,396 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595c22b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,397 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:01,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:01,399 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T04:45:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,38397,1733200986071 2024-12-03T04:45:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3b4b782b 2024-12-03T04:45:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T04:45:01,400 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T04:45:01,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:01,401 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:01,402 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752fd0b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:01,403 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2324e51b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,404 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:01,404 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:01,404 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a58005a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:01,405 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e63eb03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:01,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:01,407 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fda66ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:01,408 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40af4946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:01,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:01,409 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f927ed8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:01,411 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:01,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61037c9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:01,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:01,412 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:01,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:01,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:01,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55e3aef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:01,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:01,413 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8a04c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:01,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:01,414 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c9c336d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:01,415 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:01,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f9558a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:01,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:01,417 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2307fb8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:01,418 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1f9b20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,419 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:01,419 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:01,419 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73db9bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:01,420 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63601896, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:01,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:01,422 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b16b8ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:01,423 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9e0af7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:01,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:01,424 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c0455c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:01,425 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@464dcb2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:01,426 DEBUG [Thread-1985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:01,426 DEBUG [Thread-1989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:01,426 DEBUG [Thread-1976 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:01,426 DEBUG [Thread-1974 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:01,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:01,427 DEBUG [Thread-1980 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:01,427 DEBUG [Thread-1991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:01,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:01,428 DEBUG [Thread-1978 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:01,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-03T04:45:01,428 DEBUG [Thread-1982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:01,428 DEBUG [Thread-1987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:45:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:01,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:01,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T04:45:01,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:01,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:01,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:01,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:01,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:01,426 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:01,431 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:01,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:01,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:01,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:01,432 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:01,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:01,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:01,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:45:01,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:45:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:45:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:45:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:45:01,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:45:01,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:45:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:45:01,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2] 2024-12-03T04:45:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:01,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:45:01,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:01,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:01,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:01,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:01,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:01,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:01,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/915199c37f4542a192bbd3163b3c4ba7 is 50, key is test_row_0/A:col10/1733201101436/Put/seqid=0 2024-12-03T04:45:01,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742265_1441 (size=12001) 2024-12-03T04:45:01,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/915199c37f4542a192bbd3163b3c4ba7 2024-12-03T04:45:01,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733201101481, max timestamp=1733201101504Num uniques 859; 2024-12-03T04:45:01,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733201101481, max timestamp=1733201101504Num uniques 859; 2024-12-03T04:45:01,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733201101481, max timestamp=1733201101504Num uniques 859; 2024-12-03T04:45:01,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T04:45:01,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T04:45:01,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T04:45:01,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/382c6db10f2c4a63ab7f173c0e9db735 is 50, key is test_row_0/B:col10/1733201101436/Put/seqid=0 2024-12-03T04:45:01,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742266_1442 (size=12001) 2024-12-03T04:45:01,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/382c6db10f2c4a63ab7f173c0e9db735 2024-12-03T04:45:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T04:45:01,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/bf3f8bd61b0b4d5ea9df946e7f8b9b32 is 50, key is test_row_0/C:col10/1733201101436/Put/seqid=0 2024-12-03T04:45:01,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742267_1443 (size=12001) 2024-12-03T04:45:01,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201161538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201161539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201161540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201161542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201161554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T04:45:01,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:01,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:01,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:01,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:01,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:01,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201161665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201161665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201161665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201161666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201161675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T04:45:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:01,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T04:45:01,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201161882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201161882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T04:45:01,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:01,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:01,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:01,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:01,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:01,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201161882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201161885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201161901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:01,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/bf3f8bd61b0b4d5ea9df946e7f8b9b32 2024-12-03T04:45:01,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/915199c37f4542a192bbd3163b3c4ba7 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/915199c37f4542a192bbd3163b3c4ba7 2024-12-03T04:45:02,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/915199c37f4542a192bbd3163b3c4ba7, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T04:45:02,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/382c6db10f2c4a63ab7f173c0e9db735 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/382c6db10f2c4a63ab7f173c0e9db735 2024-12-03T04:45:02,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/382c6db10f2c4a63ab7f173c0e9db735, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T04:45:02,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/bf3f8bd61b0b4d5ea9df946e7f8b9b32 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/bf3f8bd61b0b4d5ea9df946e7f8b9b32 2024-12-03T04:45:02,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/bf3f8bd61b0b4d5ea9df946e7f8b9b32, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T04:45:02,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a9660218570dfb53504238e06a5637bb in 573ms, sequenceid=12, compaction requested=false 2024-12-03T04:45:02,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:02,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T04:45:02,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:02,050 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-03T04:45:02,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:02,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:02,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:02,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:02,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:02,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:02,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/b3917aa2601049b7b3c274363bc4536f is 50, key is test_row_0/A:col10/1733201101541/Put/seqid=0 2024-12-03T04:45:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T04:45:02,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742268_1444 (size=12001) 2024-12-03T04:45:02,105 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/b3917aa2601049b7b3c274363bc4536f 2024-12-03T04:45:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/c5fd11bf75a04b969ef80b67df9a918f is 50, key is test_row_0/B:col10/1733201101541/Put/seqid=0 2024-12-03T04:45:02,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742269_1445 (size=12001) 2024-12-03T04:45:02,135 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/c5fd11bf75a04b969ef80b67df9a918f 2024-12-03T04:45:02,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/7c95965dfb8445659c1a39dad83c727f is 50, key is test_row_0/C:col10/1733201101541/Put/seqid=0 2024-12-03T04:45:02,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742270_1446 (size=12001) 2024-12-03T04:45:02,159 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/7c95965dfb8445659c1a39dad83c727f 2024-12-03T04:45:02,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/b3917aa2601049b7b3c274363bc4536f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b3917aa2601049b7b3c274363bc4536f 2024-12-03T04:45:02,173 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b3917aa2601049b7b3c274363bc4536f, entries=150, sequenceid=46, filesize=11.7 K 2024-12-03T04:45:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/c5fd11bf75a04b969ef80b67df9a918f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c5fd11bf75a04b969ef80b67df9a918f 2024-12-03T04:45:02,177 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c5fd11bf75a04b969ef80b67df9a918f, entries=150, sequenceid=46, filesize=11.7 K 2024-12-03T04:45:02,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/7c95965dfb8445659c1a39dad83c727f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/7c95965dfb8445659c1a39dad83c727f 2024-12-03T04:45:02,182 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/7c95965dfb8445659c1a39dad83c727f, entries=150, sequenceid=46, filesize=11.7 K 2024-12-03T04:45:02,184 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=0 B/0 for a9660218570dfb53504238e06a5637bb in 134ms, sequenceid=46, compaction requested=false 2024-12-03T04:45:02,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:02,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:02,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-03T04:45:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-03T04:45:02,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-03T04:45:02,198 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 760 msec 2024-12-03T04:45:02,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 772 msec 2024-12-03T04:45:02,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:45:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:02,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:02,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:02,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:02,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:02,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:02,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:02,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/21d1e612dc7f4699a8fc594f01d5428b is 50, key is test_row_0/A:col10/1733201102209/Put/seqid=0 2024-12-03T04:45:02,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742271_1447 (size=19021) 2024-12-03T04:45:02,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201102232, max timestamp=1733201102270Num uniques 859; 2024-12-03T04:45:02,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201102232, max timestamp=1733201102269Num uniques 859; 2024-12-03T04:45:02,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201102232, max timestamp=1733201102269Num uniques 859; 2024-12-03T04:45:02,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:02,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:02,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:02,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201162324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201162326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201162328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201162329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201162337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201162452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201162452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201162452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201162452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201162461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T04:45:02,572 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:02,573 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-03T04:45:02,578 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:02,579 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:02,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T04:45:02,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/21d1e612dc7f4699a8fc594f01d5428b 2024-12-03T04:45:02,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/50f3f2caa97f4a70bc4fd4c0072e1649 is 50, key is test_row_0/B:col10/1733201102209/Put/seqid=0 2024-12-03T04:45:02,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742272_1448 (size=12001) 2024-12-03T04:45:02,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/50f3f2caa97f4a70bc4fd4c0072e1649 2024-12-03T04:45:02,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/80e7f0a2ea2b4351b34382b48d9e2ed5 is 50, key is test_row_0/C:col10/1733201102209/Put/seqid=0 2024-12-03T04:45:02,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742273_1449 (size=12001) 2024-12-03T04:45:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201162662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201162662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201162662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201162662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T04:45:02,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201162691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-03T04:45:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:02,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:02,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:02,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-03T04:45:02,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:02,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:02,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:02,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:02,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T04:45:02,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201162971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201162982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201162982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201162982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201163011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-03T04:45:03,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:03,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:03,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:03,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:03,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:03,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/80e7f0a2ea2b4351b34382b48d9e2ed5 2024-12-03T04:45:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/21d1e612dc7f4699a8fc594f01d5428b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/21d1e612dc7f4699a8fc594f01d5428b 2024-12-03T04:45:03,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/21d1e612dc7f4699a8fc594f01d5428b, entries=300, sequenceid=57, filesize=18.6 K 2024-12-03T04:45:03,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/50f3f2caa97f4a70bc4fd4c0072e1649 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/50f3f2caa97f4a70bc4fd4c0072e1649 2024-12-03T04:45:03,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/50f3f2caa97f4a70bc4fd4c0072e1649, entries=150, sequenceid=57, filesize=11.7 K 2024-12-03T04:45:03,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/80e7f0a2ea2b4351b34382b48d9e2ed5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/80e7f0a2ea2b4351b34382b48d9e2ed5 2024-12-03T04:45:03,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/80e7f0a2ea2b4351b34382b48d9e2ed5, entries=150, sequenceid=57, filesize=11.7 K 2024-12-03T04:45:03,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a9660218570dfb53504238e06a5637bb in 860ms, sequenceid=57, compaction requested=true 2024-12-03T04:45:03,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:03,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:03,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:03,073 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:03,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:03,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:03,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:03,073 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:03,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:03,074 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:03,074 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:03,074 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/A is initiating minor compaction (all files) 2024-12-03T04:45:03,074 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/B is initiating minor compaction (all files) 2024-12-03T04:45:03,074 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/A in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:03,074 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/B in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:03,074 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/915199c37f4542a192bbd3163b3c4ba7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b3917aa2601049b7b3c274363bc4536f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/21d1e612dc7f4699a8fc594f01d5428b] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=42.0 K 2024-12-03T04:45:03,074 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/382c6db10f2c4a63ab7f173c0e9db735, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c5fd11bf75a04b969ef80b67df9a918f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/50f3f2caa97f4a70bc4fd4c0072e1649] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=35.2 K 2024-12-03T04:45:03,074 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 915199c37f4542a192bbd3163b3c4ba7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733201101436 2024-12-03T04:45:03,075 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 382c6db10f2c4a63ab7f173c0e9db735, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733201101436 2024-12-03T04:45:03,075 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3917aa2601049b7b3c274363bc4536f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733201101537 2024-12-03T04:45:03,075 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c5fd11bf75a04b969ef80b67df9a918f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733201101537 2024-12-03T04:45:03,075 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21d1e612dc7f4699a8fc594f01d5428b, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733201102203 2024-12-03T04:45:03,075 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 50f3f2caa97f4a70bc4fd4c0072e1649, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733201102209 2024-12-03T04:45:03,085 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#B#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:45:03,086 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/520ec34231ba48edac8cdce467af6a17 is 50, key is test_row_0/B:col10/1733201102209/Put/seqid=0 2024-12-03T04:45:03,088 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:45:03,088 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/abdb347bf4814cdca7db3c3654e61dba is 50, key is test_row_0/A:col10/1733201102209/Put/seqid=0 2024-12-03T04:45:03,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742274_1450 (size=12104) 2024-12-03T04:45:03,148 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/520ec34231ba48edac8cdce467af6a17 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/520ec34231ba48edac8cdce467af6a17 2024-12-03T04:45:03,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742275_1451 (size=12104) 2024-12-03T04:45:03,155 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/B of a9660218570dfb53504238e06a5637bb into 520ec34231ba48edac8cdce467af6a17(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:03,155 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:03,155 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/B, priority=13, startTime=1733201103073; duration=0sec 2024-12-03T04:45:03,156 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:03,156 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:B 2024-12-03T04:45:03,156 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:03,166 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:03,166 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/C is initiating minor compaction (all files) 2024-12-03T04:45:03,166 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/C in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:03,166 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/bf3f8bd61b0b4d5ea9df946e7f8b9b32, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/7c95965dfb8445659c1a39dad83c727f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/80e7f0a2ea2b4351b34382b48d9e2ed5] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=35.2 K 2024-12-03T04:45:03,167 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting bf3f8bd61b0b4d5ea9df946e7f8b9b32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733201101436 2024-12-03T04:45:03,169 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c95965dfb8445659c1a39dad83c727f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733201101537 2024-12-03T04:45:03,169 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 80e7f0a2ea2b4351b34382b48d9e2ed5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733201102209 2024-12-03T04:45:03,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-03T04:45:03,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:03,191 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T04:45:03,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:03,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:03,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:03,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:03,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:03,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:03,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/cf5a97c62dfc48c9baf08795dbfa9310 is 50, key is test_row_0/A:col10/1733201102323/Put/seqid=0 2024-12-03T04:45:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T04:45:03,220 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#C#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:45:03,220 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/501873210b234d2d86e43f42374871d3 is 50, key is test_row_0/C:col10/1733201102209/Put/seqid=0 2024-12-03T04:45:03,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742276_1452 (size=12001) 2024-12-03T04:45:03,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742277_1453 (size=12104) 2024-12-03T04:45:03,248 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/501873210b234d2d86e43f42374871d3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/501873210b234d2d86e43f42374871d3 2024-12-03T04:45:03,252 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/C of a9660218570dfb53504238e06a5637bb into 501873210b234d2d86e43f42374871d3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:03,252 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:03,252 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/C, priority=13, startTime=1733201103073; duration=0sec 2024-12-03T04:45:03,252 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:03,252 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:C 2024-12-03T04:45:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:03,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:03,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201163518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201163522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201163523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201163524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201163532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,556 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/abdb347bf4814cdca7db3c3654e61dba as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/abdb347bf4814cdca7db3c3654e61dba 2024-12-03T04:45:03,560 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/A of a9660218570dfb53504238e06a5637bb into abdb347bf4814cdca7db3c3654e61dba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:03,560 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:03,560 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/A, priority=13, startTime=1733201103073; duration=0sec 2024-12-03T04:45:03,560 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:03,560 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:A 2024-12-03T04:45:03,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201163632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201163643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201163643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,648 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/cf5a97c62dfc48c9baf08795dbfa9310 2024-12-03T04:45:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201163645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/6455acb1dd7e4eb2a5bbf7a6a16afe56 is 50, key is test_row_0/B:col10/1733201102323/Put/seqid=0 2024-12-03T04:45:03,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742278_1454 (size=12001) 2024-12-03T04:45:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T04:45:03,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201163852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201163852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201163852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:03,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201163862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:04,074 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/6455acb1dd7e4eb2a5bbf7a6a16afe56 2024-12-03T04:45:04,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/9290cea6e0c34c479f675588945674eb is 50, key is test_row_0/C:col10/1733201102323/Put/seqid=0 2024-12-03T04:45:04,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742279_1455 (size=12001) 2024-12-03T04:45:04,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201164172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:04,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201164172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201164172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:04,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201164172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:04,367 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T04:45:04,487 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/9290cea6e0c34c479f675588945674eb 2024-12-03T04:45:04,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/cf5a97c62dfc48c9baf08795dbfa9310 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/cf5a97c62dfc48c9baf08795dbfa9310 2024-12-03T04:45:04,494 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/cf5a97c62dfc48c9baf08795dbfa9310, entries=150, sequenceid=94, filesize=11.7 K 2024-12-03T04:45:04,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/6455acb1dd7e4eb2a5bbf7a6a16afe56 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/6455acb1dd7e4eb2a5bbf7a6a16afe56 2024-12-03T04:45:04,498 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/6455acb1dd7e4eb2a5bbf7a6a16afe56, entries=150, sequenceid=94, filesize=11.7 K 2024-12-03T04:45:04,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/9290cea6e0c34c479f675588945674eb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/9290cea6e0c34c479f675588945674eb 2024-12-03T04:45:04,502 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/9290cea6e0c34c479f675588945674eb, entries=150, sequenceid=94, filesize=11.7 K 2024-12-03T04:45:04,505 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for a9660218570dfb53504238e06a5637bb in 1314ms, sequenceid=94, compaction requested=false 2024-12-03T04:45:04,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:04,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:04,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-03T04:45:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-03T04:45:04,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-03T04:45:04,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9270 sec 2024-12-03T04:45:04,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.9360 sec 2024-12-03T04:45:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:04,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:45:04,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:04,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:04,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:04,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:04,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:04,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:04,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/a36688b93f324090938a835073317287 is 50, key is test_row_0/A:col10/1733201103521/Put/seqid=0 2024-12-03T04:45:04,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742280_1456 (size=12001) 2024-12-03T04:45:04,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201164691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:04,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201164691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201164691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201164691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:04,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201164702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T04:45:04,732 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:04,732 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-03T04:45:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T04:45:04,734 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:04,734 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:04,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:04,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:04,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201164821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T04:45:04,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:04,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:04,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:04,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:04,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:04,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:04,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:04,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/a36688b93f324090938a835073317287 2024-12-03T04:45:04,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/dc23836ce90942ee8a3f83f715cbe2b5 is 50, key is test_row_0/B:col10/1733201103521/Put/seqid=0 2024-12-03T04:45:04,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742281_1457 (size=12001) 2024-12-03T04:45:05,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:05,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:05,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T04:45:05,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201165042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:05,164 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T04:45:05,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:05,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:05,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:05,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:05,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T04:45:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201165362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:05,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/dc23836ce90942ee8a3f83f715cbe2b5 2024-12-03T04:45:05,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/6a324129f11a41f385c1343053cfec31 is 50, key is test_row_0/C:col10/1733201103521/Put/seqid=0 2024-12-03T04:45:05,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742282_1458 (size=12001) 2024-12-03T04:45:05,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:05,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:05,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:05,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:05,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201165711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:05,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201165711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:05,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201165711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:05,719 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,719 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,719 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:05,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201165721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:05,733 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:05,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:05,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:05,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/6a324129f11a41f385c1343053cfec31 2024-12-03T04:45:05,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/a36688b93f324090938a835073317287 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/a36688b93f324090938a835073317287 2024-12-03T04:45:05,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/a36688b93f324090938a835073317287, entries=150, sequenceid=108, filesize=11.7 K 2024-12-03T04:45:05,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/dc23836ce90942ee8a3f83f715cbe2b5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/dc23836ce90942ee8a3f83f715cbe2b5 2024-12-03T04:45:05,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/dc23836ce90942ee8a3f83f715cbe2b5, entries=150, sequenceid=108, filesize=11.7 K 2024-12-03T04:45:05,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/6a324129f11a41f385c1343053cfec31 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/6a324129f11a41f385c1343053cfec31 2024-12-03T04:45:05,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/6a324129f11a41f385c1343053cfec31, entries=150, sequenceid=108, filesize=11.7 K 2024-12-03T04:45:05,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a9660218570dfb53504238e06a5637bb in 1269ms, sequenceid=108, compaction requested=true 2024-12-03T04:45:05,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:05,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:05,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:05,824 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:05,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:05,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:05,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:05,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:05,824 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:05,825 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:05,825 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:05,825 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/A is initiating minor compaction (all files) 2024-12-03T04:45:05,825 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/B is initiating minor compaction (all files) 2024-12-03T04:45:05,825 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/A in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,825 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/B in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,825 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/abdb347bf4814cdca7db3c3654e61dba, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/cf5a97c62dfc48c9baf08795dbfa9310, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/a36688b93f324090938a835073317287] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=35.3 K 2024-12-03T04:45:05,826 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/520ec34231ba48edac8cdce467af6a17, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/6455acb1dd7e4eb2a5bbf7a6a16afe56, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/dc23836ce90942ee8a3f83f715cbe2b5] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=35.3 K 2024-12-03T04:45:05,826 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting abdb347bf4814cdca7db3c3654e61dba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733201102209 2024-12-03T04:45:05,826 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 520ec34231ba48edac8cdce467af6a17, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733201102209 2024-12-03T04:45:05,826 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf5a97c62dfc48c9baf08795dbfa9310, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733201102323 2024-12-03T04:45:05,826 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 6455acb1dd7e4eb2a5bbf7a6a16afe56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733201102323 2024-12-03T04:45:05,826 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting dc23836ce90942ee8a3f83f715cbe2b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733201103521 2024-12-03T04:45:05,826 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting a36688b93f324090938a835073317287, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733201103521 2024-12-03T04:45:05,832 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#B#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:45:05,833 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/e58f7840e6f146dba01b9e1f515f7434 is 50, key is test_row_0/B:col10/1733201103521/Put/seqid=0 2024-12-03T04:45:05,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742283_1459 (size=12207) 2024-12-03T04:45:05,841 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:45:05,841 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/ce2b54c991164e549547f70ee5e3dbbd is 50, key is test_row_0/A:col10/1733201103521/Put/seqid=0 2024-12-03T04:45:05,844 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/e58f7840e6f146dba01b9e1f515f7434 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/e58f7840e6f146dba01b9e1f515f7434 2024-12-03T04:45:05,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742284_1460 (size=12207) 2024-12-03T04:45:05,851 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/B of a9660218570dfb53504238e06a5637bb into e58f7840e6f146dba01b9e1f515f7434(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:05,851 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:05,851 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/B, priority=13, startTime=1733201105824; duration=0sec 2024-12-03T04:45:05,852 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:05,852 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:B 2024-12-03T04:45:05,852 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:05,853 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:05,853 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/C is initiating minor compaction (all files) 2024-12-03T04:45:05,853 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/C in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,853 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/501873210b234d2d86e43f42374871d3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/9290cea6e0c34c479f675588945674eb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/6a324129f11a41f385c1343053cfec31] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=35.3 K 2024-12-03T04:45:05,854 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 501873210b234d2d86e43f42374871d3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733201102209 2024-12-03T04:45:05,854 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 9290cea6e0c34c479f675588945674eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733201102323 2024-12-03T04:45:05,854 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a324129f11a41f385c1343053cfec31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733201103521 2024-12-03T04:45:05,856 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/ce2b54c991164e549547f70ee5e3dbbd as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ce2b54c991164e549547f70ee5e3dbbd 2024-12-03T04:45:05,860 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/A of a9660218570dfb53504238e06a5637bb into ce2b54c991164e549547f70ee5e3dbbd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:05,860 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:05,860 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/A, priority=13, startTime=1733201105824; duration=0sec 2024-12-03T04:45:05,860 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:05,860 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:A 2024-12-03T04:45:05,862 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#C#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:45:05,862 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/72ffd567bfaf4992bd386ecb47af721f is 50, key is test_row_0/C:col10/1733201103521/Put/seqid=0 2024-12-03T04:45:05,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742285_1461 (size=12207) 2024-12-03T04:45:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T04:45:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:05,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T04:45:05,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:05,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:05,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:05,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:05,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:05,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:05,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/4bfd5b7580bf4d42a6e9d7273f321602 is 50, key is test_row_0/A:col10/1733201104701/Put/seqid=0 2024-12-03T04:45:05,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742286_1462 (size=14391) 2024-12-03T04:45:05,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201165950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:05,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:05,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:05,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201166061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:06,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:06,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:06,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:06,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:06,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:06,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:06,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:06,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:06,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:06,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:06,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,271 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/72ffd567bfaf4992bd386ecb47af721f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/72ffd567bfaf4992bd386ecb47af721f 2024-12-03T04:45:06,275 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/C of a9660218570dfb53504238e06a5637bb into 72ffd567bfaf4992bd386ecb47af721f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:06,275 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:06,276 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/C, priority=13, startTime=1733201105824; duration=0sec 2024-12-03T04:45:06,276 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:06,276 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:C 2024-12-03T04:45:06,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201166272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:06,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/4bfd5b7580bf4d42a6e9d7273f321602 2024-12-03T04:45:06,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/f0b25e7589fe4bce8a5d72d6a931d332 is 50, key is test_row_0/B:col10/1733201104701/Put/seqid=0 2024-12-03T04:45:06,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742287_1463 (size=12051) 2024-12-03T04:45:06,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/f0b25e7589fe4bce8a5d72d6a931d332 2024-12-03T04:45:06,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/717b6fda7da0413fa2a581d86f53e364 is 50, key is test_row_0/C:col10/1733201104701/Put/seqid=0 2024-12-03T04:45:06,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742288_1464 (size=12051) 2024-12-03T04:45:06,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:06,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:06,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:06,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:06,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:06,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:06,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:06,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:06,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201166591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:06,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:06,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:06,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:06,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/717b6fda7da0413fa2a581d86f53e364 2024-12-03T04:45:06,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:06,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:06,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:06,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:06,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/4bfd5b7580bf4d42a6e9d7273f321602 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/4bfd5b7580bf4d42a6e9d7273f321602 2024-12-03T04:45:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:06,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/4bfd5b7580bf4d42a6e9d7273f321602, entries=200, sequenceid=131, filesize=14.1 K 2024-12-03T04:45:06,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/f0b25e7589fe4bce8a5d72d6a931d332 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/f0b25e7589fe4bce8a5d72d6a931d332 2024-12-03T04:45:06,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/f0b25e7589fe4bce8a5d72d6a931d332, entries=150, sequenceid=131, filesize=11.8 K 2024-12-03T04:45:06,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/717b6fda7da0413fa2a581d86f53e364 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/717b6fda7da0413fa2a581d86f53e364 2024-12-03T04:45:06,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/717b6fda7da0413fa2a581d86f53e364, entries=150, sequenceid=131, filesize=11.8 K 2024-12-03T04:45:06,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for a9660218570dfb53504238e06a5637bb in 852ms, sequenceid=131, compaction requested=false 2024-12-03T04:45:06,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T04:45:06,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T04:45:06,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:06,883 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T04:45:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/86716c60453f468eac4a12f0fe1e77dc is 50, key is test_row_0/A:col10/1733201105949/Put/seqid=0 2024-12-03T04:45:06,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742289_1465 (size=12151) 2024-12-03T04:45:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:07,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:07,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201107224, max timestamp=1733201107273Num uniques 859; 2024-12-03T04:45:07,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:07,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201107224, max timestamp=1733201107273Num uniques 859; 2024-12-03T04:45:07,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:07,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201107224, max timestamp=1733201107273Num uniques 859; 2024-12-03T04:45:07,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:07,295 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/86716c60453f468eac4a12f0fe1e77dc 2024-12-03T04:45:07,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/3d746f6379b143bbbd4594e6afa6f5af is 50, key is test_row_0/B:col10/1733201105949/Put/seqid=0 2024-12-03T04:45:07,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742290_1466 (size=12151) 2024-12-03T04:45:07,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201167480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:07,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201167591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:07,708 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/3d746f6379b143bbbd4594e6afa6f5af 2024-12-03T04:45:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/c00f6b710dd040eb8a1ca0bf84fe1c2f is 50, key is test_row_0/C:col10/1733201105949/Put/seqid=0 2024-12-03T04:45:07,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742291_1467 (size=12151) 2024-12-03T04:45:07,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:07,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201167733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:07,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201167733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:07,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201167733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:07,746 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:07,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:07,747 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:07,747 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:07,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201167742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:07,751 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:07,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201167801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:08,129 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/c00f6b710dd040eb8a1ca0bf84fe1c2f 2024-12-03T04:45:08,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201168122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:08,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/86716c60453f468eac4a12f0fe1e77dc as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/86716c60453f468eac4a12f0fe1e77dc 2024-12-03T04:45:08,137 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/86716c60453f468eac4a12f0fe1e77dc, entries=150, sequenceid=143, filesize=11.9 K 2024-12-03T04:45:08,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/3d746f6379b143bbbd4594e6afa6f5af as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/3d746f6379b143bbbd4594e6afa6f5af 2024-12-03T04:45:08,141 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/3d746f6379b143bbbd4594e6afa6f5af, entries=150, sequenceid=143, filesize=11.9 K 2024-12-03T04:45:08,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/c00f6b710dd040eb8a1ca0bf84fe1c2f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c00f6b710dd040eb8a1ca0bf84fe1c2f 2024-12-03T04:45:08,145 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c00f6b710dd040eb8a1ca0bf84fe1c2f, entries=150, sequenceid=143, filesize=11.9 K 2024-12-03T04:45:08,145 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a9660218570dfb53504238e06a5637bb in 1262ms, sequenceid=143, compaction requested=true 2024-12-03T04:45:08,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:08,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:08,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-03T04:45:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-03T04:45:08,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-03T04:45:08,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4120 sec 2024-12-03T04:45:08,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 3.4170 sec 2024-12-03T04:45:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:08,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T04:45:08,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:08,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:08,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:08,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:08,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:08,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:08,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/f6b867bc8e91497fb5a3010af0b490b1 is 50, key is test_row_0/A:col10/1733201107420/Put/seqid=0 2024-12-03T04:45:08,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742292_1468 (size=14541) 2024-12-03T04:45:08,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/f6b867bc8e91497fb5a3010af0b490b1 2024-12-03T04:45:08,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/a03c6804077a4dfbb78599f587e9fc6f is 50, key is test_row_0/B:col10/1733201107420/Put/seqid=0 2024-12-03T04:45:08,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742293_1469 (size=12151) 2024-12-03T04:45:08,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/a03c6804077a4dfbb78599f587e9fc6f 2024-12-03T04:45:08,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/f5b5b709a5514798a4dea878303c5854 is 50, key is test_row_0/C:col10/1733201107420/Put/seqid=0 2024-12-03T04:45:08,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742294_1470 (size=12151) 2024-12-03T04:45:08,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/f5b5b709a5514798a4dea878303c5854 2024-12-03T04:45:08,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/f6b867bc8e91497fb5a3010af0b490b1 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f6b867bc8e91497fb5a3010af0b490b1 2024-12-03T04:45:08,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f6b867bc8e91497fb5a3010af0b490b1, entries=200, sequenceid=180, filesize=14.2 K 2024-12-03T04:45:08,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/a03c6804077a4dfbb78599f587e9fc6f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/a03c6804077a4dfbb78599f587e9fc6f 2024-12-03T04:45:08,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/a03c6804077a4dfbb78599f587e9fc6f, entries=150, sequenceid=180, filesize=11.9 K 2024-12-03T04:45:08,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/f5b5b709a5514798a4dea878303c5854 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/f5b5b709a5514798a4dea878303c5854 2024-12-03T04:45:08,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/f5b5b709a5514798a4dea878303c5854, entries=150, sequenceid=180, filesize=11.9 K 2024-12-03T04:45:08,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a9660218570dfb53504238e06a5637bb in 117ms, sequenceid=180, compaction requested=true 2024-12-03T04:45:08,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:08,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:08,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:08,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:08,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:08,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:08,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:08,767 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:08,767 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:08,768 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:08,768 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/B is initiating minor compaction (all files) 2024-12-03T04:45:08,768 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/B in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:08,768 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/e58f7840e6f146dba01b9e1f515f7434, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/f0b25e7589fe4bce8a5d72d6a931d332, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/3d746f6379b143bbbd4594e6afa6f5af, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/a03c6804077a4dfbb78599f587e9fc6f] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=47.4 K 2024-12-03T04:45:08,769 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:08,769 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e58f7840e6f146dba01b9e1f515f7434, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733201103521 2024-12-03T04:45:08,769 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/A is initiating minor compaction (all files) 2024-12-03T04:45:08,769 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/A in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:08,769 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ce2b54c991164e549547f70ee5e3dbbd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/4bfd5b7580bf4d42a6e9d7273f321602, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/86716c60453f468eac4a12f0fe1e77dc, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f6b867bc8e91497fb5a3010af0b490b1] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=52.0 K 2024-12-03T04:45:08,769 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting f0b25e7589fe4bce8a5d72d6a931d332, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733201104685 2024-12-03T04:45:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:08,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T04:45:08,770 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce2b54c991164e549547f70ee5e3dbbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733201103521 2024-12-03T04:45:08,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:08,770 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d746f6379b143bbbd4594e6afa6f5af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733201105922 2024-12-03T04:45:08,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:08,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:08,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:08,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:08,770 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bfd5b7580bf4d42a6e9d7273f321602, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733201104645 2024-12-03T04:45:08,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:08,771 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a03c6804077a4dfbb78599f587e9fc6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201107420 2024-12-03T04:45:08,771 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86716c60453f468eac4a12f0fe1e77dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733201105922 2024-12-03T04:45:08,771 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b867bc8e91497fb5a3010af0b490b1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201107420 2024-12-03T04:45:08,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/7682ab35da774cfd8c1ddd5c28667619 is 50, key is test_row_0/A:col10/1733201108726/Put/seqid=0 2024-12-03T04:45:08,786 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:45:08,787 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#B#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:45:08,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742295_1471 (size=14541) 2024-12-03T04:45:08,787 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/05c5ccf058ba4fd482facdf0b0f39f70 is 50, key is test_row_0/A:col10/1733201107420/Put/seqid=0 2024-12-03T04:45:08,787 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/0fa1b9b2638e4dc781d241744a13ed00 is 50, key is test_row_0/B:col10/1733201107420/Put/seqid=0 2024-12-03T04:45:08,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/7682ab35da774cfd8c1ddd5c28667619 2024-12-03T04:45:08,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742296_1472 (size=12493) 2024-12-03T04:45:08,799 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/0fa1b9b2638e4dc781d241744a13ed00 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/0fa1b9b2638e4dc781d241744a13ed00 2024-12-03T04:45:08,804 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9660218570dfb53504238e06a5637bb/B of a9660218570dfb53504238e06a5637bb into 0fa1b9b2638e4dc781d241744a13ed00(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:08,804 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:08,804 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/B, priority=12, startTime=1733201108767; duration=0sec 2024-12-03T04:45:08,804 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:08,804 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:B 2024-12-03T04:45:08,804 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:08,805 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:08,805 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/C is initiating minor compaction (all files) 2024-12-03T04:45:08,805 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/C in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:08,805 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/72ffd567bfaf4992bd386ecb47af721f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/717b6fda7da0413fa2a581d86f53e364, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c00f6b710dd040eb8a1ca0bf84fe1c2f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/f5b5b709a5514798a4dea878303c5854] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=47.4 K 2024-12-03T04:45:08,806 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ffd567bfaf4992bd386ecb47af721f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733201103521 2024-12-03T04:45:08,806 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 717b6fda7da0413fa2a581d86f53e364, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733201104685 2024-12-03T04:45:08,806 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c00f6b710dd040eb8a1ca0bf84fe1c2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733201105922 2024-12-03T04:45:08,807 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting f5b5b709a5514798a4dea878303c5854, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201107420 2024-12-03T04:45:08,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742297_1473 (size=12493) 2024-12-03T04:45:08,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/7d16c770c9204df183962baff52df763 is 50, key is test_row_0/B:col10/1733201108726/Put/seqid=0 2024-12-03T04:45:08,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742298_1474 (size=12151) 2024-12-03T04:45:08,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/7d16c770c9204df183962baff52df763 2024-12-03T04:45:08,821 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#C#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T04:45:08,822 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/9956d2128e9d4c86baac841c26050bca is 50, key is test_row_0/C:col10/1733201107420/Put/seqid=0 2024-12-03T04:45:08,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/b7983e9f9d4e4269bd1da3bb56f07568 is 50, key is test_row_0/C:col10/1733201108726/Put/seqid=0 2024-12-03T04:45:08,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742299_1475 (size=12493) 2024-12-03T04:45:08,845 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/9956d2128e9d4c86baac841c26050bca as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/9956d2128e9d4c86baac841c26050bca 2024-12-03T04:45:08,849 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9660218570dfb53504238e06a5637bb/C of a9660218570dfb53504238e06a5637bb into 9956d2128e9d4c86baac841c26050bca(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:08,849 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:08,849 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/C, priority=12, startTime=1733201108767; duration=0sec 2024-12-03T04:45:08,849 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:08,849 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:C 2024-12-03T04:45:08,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742300_1476 (size=12151) 2024-12-03T04:45:08,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/b7983e9f9d4e4269bd1da3bb56f07568 2024-12-03T04:45:08,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/7682ab35da774cfd8c1ddd5c28667619 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7682ab35da774cfd8c1ddd5c28667619 2024-12-03T04:45:08,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7682ab35da774cfd8c1ddd5c28667619, entries=200, sequenceid=191, filesize=14.2 K 2024-12-03T04:45:08,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/7d16c770c9204df183962baff52df763 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7d16c770c9204df183962baff52df763 2024-12-03T04:45:08,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7d16c770c9204df183962baff52df763, entries=150, sequenceid=191, filesize=11.9 K 2024-12-03T04:45:08,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/b7983e9f9d4e4269bd1da3bb56f07568 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/b7983e9f9d4e4269bd1da3bb56f07568 2024-12-03T04:45:08,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/b7983e9f9d4e4269bd1da3bb56f07568, entries=150, sequenceid=191, filesize=11.9 K 2024-12-03T04:45:08,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=53.67 KB/54960 for a9660218570dfb53504238e06a5637bb in 105ms, sequenceid=191, compaction requested=false 2024-12-03T04:45:08,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:08,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:45:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:08,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/f7d4a6cd1d3640c58aeea6224b651276 is 50, key is test_row_0/A:col10/1733201108851/Put/seqid=0 2024-12-03T04:45:08,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742301_1477 (size=14541) 2024-12-03T04:45:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T04:45:08,901 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:08,902 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-03T04:45:08,904 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T04:45:08,905 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:08,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T04:45:09,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201169043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:09,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T04:45:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,125 DEBUG [regionserver/32beccd76b35:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2024-12-03T04:45:09,127 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/915199c37f4542a192bbd3163b3c4ba7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b3917aa2601049b7b3c274363bc4536f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/21d1e612dc7f4699a8fc594f01d5428b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/abdb347bf4814cdca7db3c3654e61dba, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/cf5a97c62dfc48c9baf08795dbfa9310, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/a36688b93f324090938a835073317287] to archive 2024-12-03T04:45:09,128 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:45:09,130 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/915199c37f4542a192bbd3163b3c4ba7 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/915199c37f4542a192bbd3163b3c4ba7 2024-12-03T04:45:09,131 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b3917aa2601049b7b3c274363bc4536f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b3917aa2601049b7b3c274363bc4536f 2024-12-03T04:45:09,133 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/21d1e612dc7f4699a8fc594f01d5428b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/21d1e612dc7f4699a8fc594f01d5428b 2024-12-03T04:45:09,134 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/abdb347bf4814cdca7db3c3654e61dba to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/abdb347bf4814cdca7db3c3654e61dba 2024-12-03T04:45:09,135 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/cf5a97c62dfc48c9baf08795dbfa9310 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/cf5a97c62dfc48c9baf08795dbfa9310 2024-12-03T04:45:09,137 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/a36688b93f324090938a835073317287 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/a36688b93f324090938a835073317287 2024-12-03T04:45:09,139 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/382c6db10f2c4a63ab7f173c0e9db735, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c5fd11bf75a04b969ef80b67df9a918f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/520ec34231ba48edac8cdce467af6a17, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/50f3f2caa97f4a70bc4fd4c0072e1649, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/6455acb1dd7e4eb2a5bbf7a6a16afe56, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/e58f7840e6f146dba01b9e1f515f7434, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/dc23836ce90942ee8a3f83f715cbe2b5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/f0b25e7589fe4bce8a5d72d6a931d332, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/3d746f6379b143bbbd4594e6afa6f5af, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/a03c6804077a4dfbb78599f587e9fc6f] to archive 2024-12-03T04:45:09,140 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:45:09,141 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/382c6db10f2c4a63ab7f173c0e9db735 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/382c6db10f2c4a63ab7f173c0e9db735 2024-12-03T04:45:09,143 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c5fd11bf75a04b969ef80b67df9a918f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c5fd11bf75a04b969ef80b67df9a918f 2024-12-03T04:45:09,144 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/520ec34231ba48edac8cdce467af6a17 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/520ec34231ba48edac8cdce467af6a17 2024-12-03T04:45:09,145 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/50f3f2caa97f4a70bc4fd4c0072e1649 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/50f3f2caa97f4a70bc4fd4c0072e1649 2024-12-03T04:45:09,146 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/6455acb1dd7e4eb2a5bbf7a6a16afe56 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/6455acb1dd7e4eb2a5bbf7a6a16afe56 2024-12-03T04:45:09,147 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/e58f7840e6f146dba01b9e1f515f7434 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/e58f7840e6f146dba01b9e1f515f7434 2024-12-03T04:45:09,148 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/dc23836ce90942ee8a3f83f715cbe2b5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/dc23836ce90942ee8a3f83f715cbe2b5 2024-12-03T04:45:09,148 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/f0b25e7589fe4bce8a5d72d6a931d332 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/f0b25e7589fe4bce8a5d72d6a931d332 2024-12-03T04:45:09,149 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/3d746f6379b143bbbd4594e6afa6f5af to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/3d746f6379b143bbbd4594e6afa6f5af 2024-12-03T04:45:09,150 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/a03c6804077a4dfbb78599f587e9fc6f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/a03c6804077a4dfbb78599f587e9fc6f 2024-12-03T04:45:09,152 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/bf3f8bd61b0b4d5ea9df946e7f8b9b32, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/7c95965dfb8445659c1a39dad83c727f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/501873210b234d2d86e43f42374871d3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/80e7f0a2ea2b4351b34382b48d9e2ed5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/9290cea6e0c34c479f675588945674eb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/72ffd567bfaf4992bd386ecb47af721f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/6a324129f11a41f385c1343053cfec31, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/717b6fda7da0413fa2a581d86f53e364, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c00f6b710dd040eb8a1ca0bf84fe1c2f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/f5b5b709a5514798a4dea878303c5854] to archive 2024-12-03T04:45:09,153 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:45:09,154 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/bf3f8bd61b0b4d5ea9df946e7f8b9b32 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/bf3f8bd61b0b4d5ea9df946e7f8b9b32 2024-12-03T04:45:09,156 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/7c95965dfb8445659c1a39dad83c727f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/7c95965dfb8445659c1a39dad83c727f 2024-12-03T04:45:09,157 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/501873210b234d2d86e43f42374871d3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/501873210b234d2d86e43f42374871d3 2024-12-03T04:45:09,160 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/80e7f0a2ea2b4351b34382b48d9e2ed5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/80e7f0a2ea2b4351b34382b48d9e2ed5 2024-12-03T04:45:09,165 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/9290cea6e0c34c479f675588945674eb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/9290cea6e0c34c479f675588945674eb 2024-12-03T04:45:09,167 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/72ffd567bfaf4992bd386ecb47af721f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/72ffd567bfaf4992bd386ecb47af721f 2024-12-03T04:45:09,169 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/6a324129f11a41f385c1343053cfec31 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/6a324129f11a41f385c1343053cfec31 2024-12-03T04:45:09,170 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/717b6fda7da0413fa2a581d86f53e364 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/717b6fda7da0413fa2a581d86f53e364 2024-12-03T04:45:09,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201169163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:09,172 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c00f6b710dd040eb8a1ca0bf84fe1c2f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c00f6b710dd040eb8a1ca0bf84fe1c2f 2024-12-03T04:45:09,173 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32beccd76b35:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/f5b5b709a5514798a4dea878303c5854 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/f5b5b709a5514798a4dea878303c5854 2024-12-03T04:45:09,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T04:45:09,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:09,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T04:45:09,223 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/05c5ccf058ba4fd482facdf0b0f39f70 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/05c5ccf058ba4fd482facdf0b0f39f70 2024-12-03T04:45:09,227 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9660218570dfb53504238e06a5637bb/A of a9660218570dfb53504238e06a5637bb into 05c5ccf058ba4fd482facdf0b0f39f70(size=12.2 K), total size for store is 26.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:09,227 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:09,227 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/A, priority=12, startTime=1733201108767; duration=0sec 2024-12-03T04:45:09,227 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:09,227 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:A 2024-12-03T04:45:09,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/f7d4a6cd1d3640c58aeea6224b651276 2024-12-03T04:45:09,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/abe513bbed4148e199aae94f378cb1da is 50, key is test_row_0/B:col10/1733201108851/Put/seqid=0 2024-12-03T04:45:09,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742302_1478 (size=12151) 2024-12-03T04:45:09,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/abe513bbed4148e199aae94f378cb1da 2024-12-03T04:45:09,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/cdc4a1a0439e4b60b79420203d5e0425 is 50, key is test_row_0/C:col10/1733201108851/Put/seqid=0 2024-12-03T04:45:09,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742303_1479 (size=12151) 2024-12-03T04:45:09,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/cdc4a1a0439e4b60b79420203d5e0425 2024-12-03T04:45:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/f7d4a6cd1d3640c58aeea6224b651276 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f7d4a6cd1d3640c58aeea6224b651276 2024-12-03T04:45:09,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f7d4a6cd1d3640c58aeea6224b651276, entries=200, sequenceid=205, filesize=14.2 K 2024-12-03T04:45:09,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/abe513bbed4148e199aae94f378cb1da as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/abe513bbed4148e199aae94f378cb1da 2024-12-03T04:45:09,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/abe513bbed4148e199aae94f378cb1da, entries=150, sequenceid=205, filesize=11.9 K 2024-12-03T04:45:09,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/cdc4a1a0439e4b60b79420203d5e0425 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/cdc4a1a0439e4b60b79420203d5e0425 2024-12-03T04:45:09,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/cdc4a1a0439e4b60b79420203d5e0425, entries=150, sequenceid=205, filesize=11.9 K 2024-12-03T04:45:09,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a9660218570dfb53504238e06a5637bb in 452ms, sequenceid=205, compaction requested=true 2024-12-03T04:45:09,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:09,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:09,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:09,332 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:09,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:09,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:09,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:09,332 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:09,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:09,333 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:09,333 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:09,333 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/A is initiating minor compaction (all files) 2024-12-03T04:45:09,333 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/B is initiating minor compaction (all files) 2024-12-03T04:45:09,333 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/A in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,333 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/B in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,333 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/0fa1b9b2638e4dc781d241744a13ed00, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7d16c770c9204df183962baff52df763, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/abe513bbed4148e199aae94f378cb1da] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=35.9 K 2024-12-03T04:45:09,333 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/05c5ccf058ba4fd482facdf0b0f39f70, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7682ab35da774cfd8c1ddd5c28667619, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f7d4a6cd1d3640c58aeea6224b651276] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=40.6 K 2024-12-03T04:45:09,333 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c5ccf058ba4fd482facdf0b0f39f70, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201107420 2024-12-03T04:45:09,333 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fa1b9b2638e4dc781d241744a13ed00, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201107420 2024-12-03T04:45:09,334 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7682ab35da774cfd8c1ddd5c28667619, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733201108677 2024-12-03T04:45:09,334 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d16c770c9204df183962baff52df763, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733201108726 2024-12-03T04:45:09,334 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7d4a6cd1d3640c58aeea6224b651276, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733201108807 2024-12-03T04:45:09,334 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting abe513bbed4148e199aae94f378cb1da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733201108807 2024-12-03T04:45:09,341 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#A#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:09,341 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#B#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:09,341 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/148b9d51180d4d8594719a5c018c703c is 50, key is test_row_0/A:col10/1733201108851/Put/seqid=0 2024-12-03T04:45:09,341 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/5ae4ea290c9e458c996ea959c6f990e2 is 50, key is test_row_0/B:col10/1733201108851/Put/seqid=0 2024-12-03T04:45:09,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742304_1480 (size=12391) 2024-12-03T04:45:09,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742305_1481 (size=12254) 2024-12-03T04:45:09,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T04:45:09,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,364 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:45:09,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:09,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:09,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:09,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:09,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:09,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/f21d597a973740e295a91d64c5c047e2 is 50, key is test_row_0/A:col10/1733201109034/Put/seqid=0 2024-12-03T04:45:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:09,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:09,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742306_1482 (size=12151) 2024-12-03T04:45:09,391 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/f21d597a973740e295a91d64c5c047e2 2024-12-03T04:45:09,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/b34b2952b5a04b769be74ea300a9727f is 50, key is test_row_0/B:col10/1733201109034/Put/seqid=0 2024-12-03T04:45:09,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742307_1483 (size=12151) 2024-12-03T04:45:09,418 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/b34b2952b5a04b769be74ea300a9727f 2024-12-03T04:45:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/8076e12e3c554ae89f8bde24fcd818a4 is 50, key is test_row_0/C:col10/1733201109034/Put/seqid=0 2024-12-03T04:45:09,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742308_1484 (size=12151) 2024-12-03T04:45:09,443 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/8076e12e3c554ae89f8bde24fcd818a4 2024-12-03T04:45:09,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/f21d597a973740e295a91d64c5c047e2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f21d597a973740e295a91d64c5c047e2 2024-12-03T04:45:09,464 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f21d597a973740e295a91d64c5c047e2, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T04:45:09,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/b34b2952b5a04b769be74ea300a9727f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/b34b2952b5a04b769be74ea300a9727f 2024-12-03T04:45:09,472 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/b34b2952b5a04b769be74ea300a9727f, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T04:45:09,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/8076e12e3c554ae89f8bde24fcd818a4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/8076e12e3c554ae89f8bde24fcd818a4 2024-12-03T04:45:09,477 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/8076e12e3c554ae89f8bde24fcd818a4, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T04:45:09,477 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=53.67 KB/54960 for a9660218570dfb53504238e06a5637bb in 113ms, sequenceid=226, compaction requested=true 2024-12-03T04:45:09,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:09,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-03T04:45:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-03T04:45:09,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-03T04:45:09,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 574 msec 2024-12-03T04:45:09,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 581 msec 2024-12-03T04:45:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:09,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:45:09,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:09,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:09,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:09,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:09,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:09,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:09,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/e2ea407d79c542a2a8f6e9cf9f660e84 is 50, key is test_row_0/A:col10/1733201109490/Put/seqid=0 2024-12-03T04:45:09,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742309_1485 (size=14541) 2024-12-03T04:45:09,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/e2ea407d79c542a2a8f6e9cf9f660e84 2024-12-03T04:45:09,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/99d6f41cda7a49e49b96f16d357c4bef is 50, key is test_row_0/B:col10/1733201109490/Put/seqid=0 2024-12-03T04:45:09,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742310_1486 (size=12151) 2024-12-03T04:45:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T04:45:09,532 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:09,532 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-03T04:45:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T04:45:09,534 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:09,535 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:09,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T04:45:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201169662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:09,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T04:45:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:09,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,750 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/5ae4ea290c9e458c996ea959c6f990e2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/5ae4ea290c9e458c996ea959c6f990e2 2024-12-03T04:45:09,751 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/148b9d51180d4d8594719a5c018c703c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/148b9d51180d4d8594719a5c018c703c 2024-12-03T04:45:09,757 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/B of a9660218570dfb53504238e06a5637bb into 5ae4ea290c9e458c996ea959c6f990e2(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:09,757 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/A of a9660218570dfb53504238e06a5637bb into 148b9d51180d4d8594719a5c018c703c(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:09,757 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:09,757 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:09,757 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/A, priority=13, startTime=1733201109332; duration=0sec 2024-12-03T04:45:09,757 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/B, priority=13, startTime=1733201109332; duration=0sec 2024-12-03T04:45:09,757 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:09,757 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:A 2024-12-03T04:45:09,757 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:09,757 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:B 2024-12-03T04:45:09,757 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:09,758 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:09,758 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/C is initiating minor compaction (all files) 2024-12-03T04:45:09,758 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/C in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,758 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/9956d2128e9d4c86baac841c26050bca, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/b7983e9f9d4e4269bd1da3bb56f07568, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/cdc4a1a0439e4b60b79420203d5e0425, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/8076e12e3c554ae89f8bde24fcd818a4] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=47.8 K 2024-12-03T04:45:09,759 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9956d2128e9d4c86baac841c26050bca, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201107420 2024-12-03T04:45:09,759 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7983e9f9d4e4269bd1da3bb56f07568, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733201108726 2024-12-03T04:45:09,759 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdc4a1a0439e4b60b79420203d5e0425, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733201108807 2024-12-03T04:45:09,760 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8076e12e3c554ae89f8bde24fcd818a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733201109012 2024-12-03T04:45:09,767 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:09,768 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/df397e3a34f1423fb1a6bb6d86bc5e8a is 50, key is test_row_0/C:col10/1733201109034/Put/seqid=0 2024-12-03T04:45:09,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742311_1487 (size=12289) 2024-12-03T04:45:09,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201169782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:09,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T04:45:09,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:09,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:09,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T04:45:09,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/99d6f41cda7a49e49b96f16d357c4bef 2024-12-03T04:45:09,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/3ae84e94916c4c6eb191b555d337972e is 50, key is test_row_0/C:col10/1733201109490/Put/seqid=0 2024-12-03T04:45:09,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T04:45:10,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:10,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742312_1488 (size=12151) 2024-12-03T04:45:10,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:10,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:10,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:10,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:10,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/3ae84e94916c4c6eb191b555d337972e 2024-12-03T04:45:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:10,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/e2ea407d79c542a2a8f6e9cf9f660e84 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e2ea407d79c542a2a8f6e9cf9f660e84 2024-12-03T04:45:10,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201170002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:10,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e2ea407d79c542a2a8f6e9cf9f660e84, entries=200, sequenceid=238, filesize=14.2 K 2024-12-03T04:45:10,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/99d6f41cda7a49e49b96f16d357c4bef as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/99d6f41cda7a49e49b96f16d357c4bef 2024-12-03T04:45:10,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/99d6f41cda7a49e49b96f16d357c4bef, entries=150, sequenceid=238, filesize=11.9 K 2024-12-03T04:45:10,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/3ae84e94916c4c6eb191b555d337972e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ae84e94916c4c6eb191b555d337972e 2024-12-03T04:45:10,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ae84e94916c4c6eb191b555d337972e, entries=150, sequenceid=238, filesize=11.9 K 2024-12-03T04:45:10,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a9660218570dfb53504238e06a5637bb in 544ms, sequenceid=238, compaction requested=true 2024-12-03T04:45:10,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:10,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:A, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:10,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:10,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:B, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:10,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T04:45:10,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:10,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T04:45:10,041 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-12-03T04:45:10,043 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T04:45:10,043 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T04:45:10,043 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. because compaction request was cancelled 2024-12-03T04:45:10,043 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:C 2024-12-03T04:45:10,044 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:10,044 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:10,044 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/A is initiating minor compaction (all files) 2024-12-03T04:45:10,044 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/A in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:10,044 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/148b9d51180d4d8594719a5c018c703c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f21d597a973740e295a91d64c5c047e2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e2ea407d79c542a2a8f6e9cf9f660e84] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=38.2 K 2024-12-03T04:45:10,045 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 148b9d51180d4d8594719a5c018c703c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733201108807 2024-12-03T04:45:10,046 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting f21d597a973740e295a91d64c5c047e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733201109012 2024-12-03T04:45:10,046 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting e2ea407d79c542a2a8f6e9cf9f660e84, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733201109444 2024-12-03T04:45:10,053 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#A#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:10,053 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/90d02793deb049c383fc1b99cd92a64f is 50, key is test_row_0/A:col10/1733201109490/Put/seqid=0 2024-12-03T04:45:10,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742313_1489 (size=12493) 2024-12-03T04:45:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T04:45:10,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T04:45:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:10,172 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:45:10,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:10,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:10,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:10,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:10,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:10,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:10,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/aa551881c77f47f99bc2de529bf4493b is 50, key is test_row_0/A:col10/1733201109654/Put/seqid=0 2024-12-03T04:45:10,204 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/df397e3a34f1423fb1a6bb6d86bc5e8a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/df397e3a34f1423fb1a6bb6d86bc5e8a 2024-12-03T04:45:10,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742314_1490 (size=12201) 2024-12-03T04:45:10,229 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/aa551881c77f47f99bc2de529bf4493b 2024-12-03T04:45:10,230 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9660218570dfb53504238e06a5637bb/C of a9660218570dfb53504238e06a5637bb into df397e3a34f1423fb1a6bb6d86bc5e8a(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:10,230 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:10,230 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/C, priority=12, startTime=1733201109332; duration=0sec 2024-12-03T04:45:10,231 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:10,231 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:C 2024-12-03T04:45:10,231 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:10,237 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:10,237 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/B is initiating minor compaction (all files) 2024-12-03T04:45:10,237 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/B in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:10,237 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/5ae4ea290c9e458c996ea959c6f990e2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/b34b2952b5a04b769be74ea300a9727f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/99d6f41cda7a49e49b96f16d357c4bef] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=35.7 K 2024-12-03T04:45:10,240 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ae4ea290c9e458c996ea959c6f990e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733201108807 2024-12-03T04:45:10,246 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting b34b2952b5a04b769be74ea300a9727f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733201109012 2024-12-03T04:45:10,248 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99d6f41cda7a49e49b96f16d357c4bef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733201109444 2024-12-03T04:45:10,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/18a67a99d1e949d3a70cd1c8438d5a4f is 50, key is test_row_0/B:col10/1733201109654/Put/seqid=0 2024-12-03T04:45:10,273 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#B#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:10,273 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/7e95852ed9f74f3facb8c245f61302df is 50, key is test_row_0/B:col10/1733201109490/Put/seqid=0 2024-12-03T04:45:10,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742315_1491 (size=12201) 2024-12-03T04:45:10,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742316_1492 (size=12357) 2024-12-03T04:45:10,297 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/18a67a99d1e949d3a70cd1c8438d5a4f 2024-12-03T04:45:10,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/eb8d7a56ec304f009199a32c8188a881 is 50, key is test_row_0/C:col10/1733201109654/Put/seqid=0 2024-12-03T04:45:10,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742317_1493 (size=12201) 2024-12-03T04:45:10,321 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/eb8d7a56ec304f009199a32c8188a881 2024-12-03T04:45:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/aa551881c77f47f99bc2de529bf4493b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/aa551881c77f47f99bc2de529bf4493b 2024-12-03T04:45:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:10,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:10,334 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/aa551881c77f47f99bc2de529bf4493b, entries=150, sequenceid=260, filesize=11.9 K 2024-12-03T04:45:10,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/18a67a99d1e949d3a70cd1c8438d5a4f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/18a67a99d1e949d3a70cd1c8438d5a4f 2024-12-03T04:45:10,339 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/18a67a99d1e949d3a70cd1c8438d5a4f, entries=150, sequenceid=260, filesize=11.9 K 2024-12-03T04:45:10,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/eb8d7a56ec304f009199a32c8188a881 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/eb8d7a56ec304f009199a32c8188a881 2024-12-03T04:45:10,344 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/eb8d7a56ec304f009199a32c8188a881, entries=150, sequenceid=260, filesize=11.9 K 2024-12-03T04:45:10,346 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for a9660218570dfb53504238e06a5637bb in 173ms, sequenceid=260, compaction requested=true 2024-12-03T04:45:10,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:10,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:10,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-03T04:45:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-03T04:45:10,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-03T04:45:10,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 817 msec 2024-12-03T04:45:10,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 823 msec 2024-12-03T04:45:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:10,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:45:10,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:10,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:10,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:10,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:10,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:10,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:10,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/7f1c4d50ba8440bd9e5e3296114f887f is 50, key is test_row_0/A:col10/1733201110362/Put/seqid=0 2024-12-03T04:45:10,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742318_1494 (size=14741) 2024-12-03T04:45:10,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/7f1c4d50ba8440bd9e5e3296114f887f 2024-12-03T04:45:10,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/c44e84b997b0496f9aadef25fbbc2d86 is 50, key is test_row_0/B:col10/1733201110362/Put/seqid=0 2024-12-03T04:45:10,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742319_1495 (size=12301) 2024-12-03T04:45:10,477 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/90d02793deb049c383fc1b99cd92a64f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/90d02793deb049c383fc1b99cd92a64f 2024-12-03T04:45:10,482 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/A of a9660218570dfb53504238e06a5637bb into 90d02793deb049c383fc1b99cd92a64f(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:10,482 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:10,482 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/A, priority=13, startTime=1733201110038; duration=0sec 2024-12-03T04:45:10,482 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:10,482 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:A 2024-12-03T04:45:10,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201110510, max timestamp=1733201110572Num uniques 859; 2024-12-03T04:45:10,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:10,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201110510, max timestamp=1733201110572Num uniques 859; 2024-12-03T04:45:10,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:10,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201110510, max timestamp=1733201110572Num uniques 859; 2024-12-03T04:45:10,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T04:45:10,672 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:10,672 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-03T04:45:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T04:45:10,674 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:10,674 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:10,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:10,699 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/7e95852ed9f74f3facb8c245f61302df as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7e95852ed9f74f3facb8c245f61302df 2024-12-03T04:45:10,703 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/B of a9660218570dfb53504238e06a5637bb into 7e95852ed9f74f3facb8c245f61302df(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:10,703 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:10,703 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/B, priority=13, startTime=1733201110038; duration=0sec 2024-12-03T04:45:10,703 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:10,703 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:B 2024-12-03T04:45:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T04:45:10,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/c44e84b997b0496f9aadef25fbbc2d86 2024-12-03T04:45:10,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201170798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:10,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/1c5b907b070743229892fd2e17701504 is 50, key is test_row_0/C:col10/1733201110362/Put/seqid=0 2024-12-03T04:45:10,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742320_1496 (size=12301) 2024-12-03T04:45:10,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T04:45:10,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:10,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:10,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:10,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:10,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:10,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201170922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:10,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T04:45:10,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:10,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:10,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:10,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:10,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T04:45:11,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T04:45:11,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:11,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:11,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:11,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:11,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:11,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201171142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:11,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/1c5b907b070743229892fd2e17701504 2024-12-03T04:45:11,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/7f1c4d50ba8440bd9e5e3296114f887f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7f1c4d50ba8440bd9e5e3296114f887f 2024-12-03T04:45:11,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7f1c4d50ba8440bd9e5e3296114f887f, entries=200, sequenceid=271, filesize=14.4 K 2024-12-03T04:45:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/c44e84b997b0496f9aadef25fbbc2d86 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c44e84b997b0496f9aadef25fbbc2d86 2024-12-03T04:45:11,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c44e84b997b0496f9aadef25fbbc2d86, entries=150, sequenceid=271, filesize=12.0 K 2024-12-03T04:45:11,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/1c5b907b070743229892fd2e17701504 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1c5b907b070743229892fd2e17701504 2024-12-03T04:45:11,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1c5b907b070743229892fd2e17701504, entries=150, sequenceid=271, filesize=12.0 K 2024-12-03T04:45:11,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a9660218570dfb53504238e06a5637bb in 864ms, sequenceid=271, compaction requested=true 2024-12-03T04:45:11,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:11,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:11,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:11,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:11,237 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:11,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:11,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:11,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:11,237 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:11,239 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36859 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:11,239 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:11,239 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/B is initiating minor compaction (all files) 2024-12-03T04:45:11,239 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/A is initiating minor compaction (all files) 2024-12-03T04:45:11,239 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/B in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:11,239 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/A in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:11,239 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7e95852ed9f74f3facb8c245f61302df, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/18a67a99d1e949d3a70cd1c8438d5a4f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c44e84b997b0496f9aadef25fbbc2d86] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=36.0 K 2024-12-03T04:45:11,239 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/90d02793deb049c383fc1b99cd92a64f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/aa551881c77f47f99bc2de529bf4493b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7f1c4d50ba8440bd9e5e3296114f887f] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=38.5 K 2024-12-03T04:45:11,239 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e95852ed9f74f3facb8c245f61302df, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733201109444 2024-12-03T04:45:11,239 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90d02793deb049c383fc1b99cd92a64f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733201109444 2024-12-03T04:45:11,240 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a67a99d1e949d3a70cd1c8438d5a4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733201109636 2024-12-03T04:45:11,240 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa551881c77f47f99bc2de529bf4493b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733201109636 2024-12-03T04:45:11,240 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c44e84b997b0496f9aadef25fbbc2d86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733201110362 2024-12-03T04:45:11,240 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f1c4d50ba8440bd9e5e3296114f887f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733201110352 2024-12-03T04:45:11,247 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:11,247 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/1e3264fa8d7e47aeaa2b9d2fc63c6b1b is 50, key is test_row_0/B:col10/1733201110362/Put/seqid=0 2024-12-03T04:45:11,250 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#A#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:11,250 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/ea013109c33d4ffeab362f07cc323608 is 50, key is test_row_0/A:col10/1733201110362/Put/seqid=0 2024-12-03T04:45:11,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742321_1497 (size=12609) 2024-12-03T04:45:11,259 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/1e3264fa8d7e47aeaa2b9d2fc63c6b1b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/1e3264fa8d7e47aeaa2b9d2fc63c6b1b 2024-12-03T04:45:11,263 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/B of a9660218570dfb53504238e06a5637bb into 1e3264fa8d7e47aeaa2b9d2fc63c6b1b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:11,263 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:11,263 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/B, priority=13, startTime=1733201111237; duration=0sec 2024-12-03T04:45:11,263 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:11,263 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:B 2024-12-03T04:45:11,263 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:11,265 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48942 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:11,267 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/C is initiating minor compaction (all files) 2024-12-03T04:45:11,267 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/C in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:11,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742322_1498 (size=12745) 2024-12-03T04:45:11,267 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/df397e3a34f1423fb1a6bb6d86bc5e8a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ae84e94916c4c6eb191b555d337972e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/eb8d7a56ec304f009199a32c8188a881, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1c5b907b070743229892fd2e17701504] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=47.8 K 2024-12-03T04:45:11,268 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting df397e3a34f1423fb1a6bb6d86bc5e8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733201109012 2024-12-03T04:45:11,268 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ae84e94916c4c6eb191b555d337972e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733201109444 2024-12-03T04:45:11,268 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting eb8d7a56ec304f009199a32c8188a881, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733201109636 2024-12-03T04:45:11,268 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c5b907b070743229892fd2e17701504, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733201110362 2024-12-03T04:45:11,278 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:11,279 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/e048586513ef44a68d9152c54fb7820c is 50, key is test_row_0/C:col10/1733201110362/Put/seqid=0 2024-12-03T04:45:11,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742323_1499 (size=12575) 2024-12-03T04:45:11,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T04:45:11,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:11,289 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T04:45:11,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:11,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:11,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:11,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:11,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:11,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:11,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/8c71a0b6907f47ad8dae7a56c116023a is 50, key is test_row_0/A:col10/1733201110662/Put/seqid=0 2024-12-03T04:45:11,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742324_1500 (size=12301) 2024-12-03T04:45:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T04:45:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:11,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:11,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201171544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:11,679 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/ea013109c33d4ffeab362f07cc323608 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ea013109c33d4ffeab362f07cc323608 2024-12-03T04:45:11,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201171665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:11,684 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/A of a9660218570dfb53504238e06a5637bb into ea013109c33d4ffeab362f07cc323608(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:11,684 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:11,684 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/A, priority=13, startTime=1733201111237; duration=0sec 2024-12-03T04:45:11,684 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:11,684 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:A 2024-12-03T04:45:11,697 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/e048586513ef44a68d9152c54fb7820c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/e048586513ef44a68d9152c54fb7820c 2024-12-03T04:45:11,701 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/8c71a0b6907f47ad8dae7a56c116023a 2024-12-03T04:45:11,715 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9660218570dfb53504238e06a5637bb/C of a9660218570dfb53504238e06a5637bb into e048586513ef44a68d9152c54fb7820c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:11,716 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:11,716 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/C, priority=12, startTime=1733201111237; duration=0sec 2024-12-03T04:45:11,716 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:11,716 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:C 2024-12-03T04:45:11,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/274420c727394b9198b386e51dcd1f70 is 50, key is test_row_0/B:col10/1733201110662/Put/seqid=0 2024-12-03T04:45:11,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742325_1501 (size=12301) 2024-12-03T04:45:11,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733201171751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:11,764 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_1' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:11,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38648 deadline: 1733201171772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:11,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38656 deadline: 1733201171772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:11,780 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:11,780 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:11,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38672 deadline: 1733201171781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:11,794 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in a9660218570dfb53504238e06a5637bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T04:45:11,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201171891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:11,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:12,136 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/274420c727394b9198b386e51dcd1f70 2024-12-03T04:45:12,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/61cda50394cb404ab90de76b83f3b9b7 is 50, key is test_row_0/C:col10/1733201110662/Put/seqid=0 2024-12-03T04:45:12,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742326_1502 (size=12301) 2024-12-03T04:45:12,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201172212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:12,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:12,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:12,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:12,553 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/61cda50394cb404ab90de76b83f3b9b7 2024-12-03T04:45:12,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/8c71a0b6907f47ad8dae7a56c116023a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8c71a0b6907f47ad8dae7a56c116023a 2024-12-03T04:45:12,564 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8c71a0b6907f47ad8dae7a56c116023a, entries=150, sequenceid=311, filesize=12.0 K 2024-12-03T04:45:12,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/274420c727394b9198b386e51dcd1f70 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/274420c727394b9198b386e51dcd1f70 2024-12-03T04:45:12,568 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/274420c727394b9198b386e51dcd1f70, entries=150, sequenceid=311, filesize=12.0 K 2024-12-03T04:45:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/61cda50394cb404ab90de76b83f3b9b7 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/61cda50394cb404ab90de76b83f3b9b7 2024-12-03T04:45:12,571 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/61cda50394cb404ab90de76b83f3b9b7, entries=150, sequenceid=311, filesize=12.0 K 2024-12-03T04:45:12,572 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a9660218570dfb53504238e06a5637bb in 1283ms, sequenceid=311, compaction requested=false 2024-12-03T04:45:12,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:12,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:12,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-03T04:45:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-03T04:45:12,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-03T04:45:12,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9010 sec 2024-12-03T04:45:12,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.9070 sec 2024-12-03T04:45:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:12,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T04:45:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:12,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/e68be19b1cf44eba98ab1a90c79c30cc is 50, key is test_row_0/A:col10/1733201111492/Put/seqid=0 2024-12-03T04:45:12,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742327_1503 (size=14741) 2024-12-03T04:45:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T04:45:12,822 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:12,822 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-03T04:45:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T04:45:12,823 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:12,824 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:12,824 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:12,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201112743, max timestamp=1733201112883Num uniques 859; 2024-12-03T04:45:12,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201112743, max timestamp=1733201112883Num uniques 859; 2024-12-03T04:45:12,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201112743, max timestamp=1733201112883Num uniques 859; 2024-12-03T04:45:12,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:12,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:12,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:12,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T04:45:12,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T04:45:12,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:12,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:12,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:12,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:12,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:13,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201173046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:13,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T04:45:13,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:13,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:13,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:13,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:13,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:13,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T04:45:13,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/e68be19b1cf44eba98ab1a90c79c30cc 2024-12-03T04:45:13,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/c615d7e3e73c4574a9c91e11349f4e92 is 50, key is test_row_0/B:col10/1733201111492/Put/seqid=0 2024-12-03T04:45:13,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742328_1504 (size=12301) 2024-12-03T04:45:13,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/c615d7e3e73c4574a9c91e11349f4e92 2024-12-03T04:45:13,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201173152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:13,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/c933cb3d05ce4bb7a4d347ec8e909c49 is 50, key is test_row_0/C:col10/1733201111492/Put/seqid=0 2024-12-03T04:45:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:13,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742329_1505 (size=12301) 2024-12-03T04:45:13,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T04:45:13,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:13,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:13,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:13,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:13,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:13,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201173371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:13,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T04:45:13,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:13,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:13,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:13,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:13,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T04:45:13,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/c933cb3d05ce4bb7a4d347ec8e909c49 2024-12-03T04:45:13,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/e68be19b1cf44eba98ab1a90c79c30cc as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e68be19b1cf44eba98ab1a90c79c30cc 2024-12-03T04:45:13,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e68be19b1cf44eba98ab1a90c79c30cc, entries=200, sequenceid=324, filesize=14.4 K 2024-12-03T04:45:13,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/c615d7e3e73c4574a9c91e11349f4e92 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c615d7e3e73c4574a9c91e11349f4e92 2024-12-03T04:45:13,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c615d7e3e73c4574a9c91e11349f4e92, entries=150, sequenceid=324, filesize=12.0 K 2024-12-03T04:45:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/c933cb3d05ce4bb7a4d347ec8e909c49 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c933cb3d05ce4bb7a4d347ec8e909c49 2024-12-03T04:45:13,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c933cb3d05ce4bb7a4d347ec8e909c49, entries=150, sequenceid=324, filesize=12.0 K 2024-12-03T04:45:13,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a9660218570dfb53504238e06a5637bb in 847ms, sequenceid=324, compaction requested=true 2024-12-03T04:45:13,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:13,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:13,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:13,583 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:13,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:13,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:13,583 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:13,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:13,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:13,584 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:13,584 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/A is initiating minor compaction (all files) 2024-12-03T04:45:13,584 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:13,584 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/A in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:13,584 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/B is initiating minor compaction (all files) 2024-12-03T04:45:13,584 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/B in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:13,584 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ea013109c33d4ffeab362f07cc323608, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8c71a0b6907f47ad8dae7a56c116023a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e68be19b1cf44eba98ab1a90c79c30cc] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=38.9 K 2024-12-03T04:45:13,584 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/1e3264fa8d7e47aeaa2b9d2fc63c6b1b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/274420c727394b9198b386e51dcd1f70, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c615d7e3e73c4574a9c91e11349f4e92] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=36.3 K 2024-12-03T04:45:13,584 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea013109c33d4ffeab362f07cc323608, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733201110362 2024-12-03T04:45:13,584 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e3264fa8d7e47aeaa2b9d2fc63c6b1b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733201110362 2024-12-03T04:45:13,584 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 274420c727394b9198b386e51dcd1f70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733201110662 2024-12-03T04:45:13,584 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c71a0b6907f47ad8dae7a56c116023a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733201110662 2024-12-03T04:45:13,585 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c615d7e3e73c4574a9c91e11349f4e92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733201111492 2024-12-03T04:45:13,585 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting e68be19b1cf44eba98ab1a90c79c30cc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733201111492 2024-12-03T04:45:13,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T04:45:13,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:13,588 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T04:45:13,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:13,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:13,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:13,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:13,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:13,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:13,593 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#B#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:13,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/bdaa5add4eed4437b7ea3a7a50281695 is 50, key is test_row_0/A:col10/1733201112969/Put/seqid=0 2024-12-03T04:45:13,593 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/cc9562829a364e4298eb5c7d1d7ceae4 is 50, key is test_row_0/B:col10/1733201111492/Put/seqid=0 2024-12-03T04:45:13,596 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:13,596 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/1d23552ca34d42a584a2598d4e197236 is 50, key is test_row_0/A:col10/1733201111492/Put/seqid=0 2024-12-03T04:45:13,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742330_1506 (size=12301) 2024-12-03T04:45:13,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742332_1508 (size=12711) 2024-12-03T04:45:13,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742331_1507 (size=12847) 2024-12-03T04:45:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:13,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:13,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201173770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:13,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201173881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:13,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T04:45:14,000 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/bdaa5add4eed4437b7ea3a7a50281695 2024-12-03T04:45:14,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/66995955a24e4ce1a7cb90d7550e0d6c is 50, key is test_row_0/B:col10/1733201112969/Put/seqid=0 2024-12-03T04:45:14,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742333_1509 (size=12301) 2024-12-03T04:45:14,015 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/1d23552ca34d42a584a2598d4e197236 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/1d23552ca34d42a584a2598d4e197236 2024-12-03T04:45:14,015 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/66995955a24e4ce1a7cb90d7550e0d6c 2024-12-03T04:45:14,017 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/cc9562829a364e4298eb5c7d1d7ceae4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/cc9562829a364e4298eb5c7d1d7ceae4 2024-12-03T04:45:14,025 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/A of a9660218570dfb53504238e06a5637bb into 1d23552ca34d42a584a2598d4e197236(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:14,025 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:14,025 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/A, priority=13, startTime=1733201113583; duration=0sec 2024-12-03T04:45:14,025 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:14,025 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:A 2024-12-03T04:45:14,025 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:14,026 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:14,026 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/C is initiating minor compaction (all files) 2024-12-03T04:45:14,026 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/C in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:14,026 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/e048586513ef44a68d9152c54fb7820c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/61cda50394cb404ab90de76b83f3b9b7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c933cb3d05ce4bb7a4d347ec8e909c49] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=36.3 K 2024-12-03T04:45:14,027 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting e048586513ef44a68d9152c54fb7820c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733201110362 2024-12-03T04:45:14,027 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61cda50394cb404ab90de76b83f3b9b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733201110662 2024-12-03T04:45:14,027 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting c933cb3d05ce4bb7a4d347ec8e909c49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733201111492 2024-12-03T04:45:14,028 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/B of a9660218570dfb53504238e06a5637bb into cc9562829a364e4298eb5c7d1d7ceae4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:14,028 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:14,028 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/B, priority=13, startTime=1733201113583; duration=0sec 2024-12-03T04:45:14,028 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:14,028 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:B 2024-12-03T04:45:14,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/28d04c31551c4ad98c8888deb3a596aa is 50, key is test_row_0/C:col10/1733201112969/Put/seqid=0 2024-12-03T04:45:14,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742334_1510 (size=12301) 2024-12-03T04:45:14,038 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#C#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:14,038 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/0bc6345f6c6f4252926799b97563ff07 is 50, key is test_row_0/C:col10/1733201111492/Put/seqid=0 2024-12-03T04:45:14,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742335_1511 (size=12677) 2024-12-03T04:45:14,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201174101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:14,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:14,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:14,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:14,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201174421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:14,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:14,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:14,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:14,436 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/28d04c31551c4ad98c8888deb3a596aa 2024-12-03T04:45:14,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/bdaa5add4eed4437b7ea3a7a50281695 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/bdaa5add4eed4437b7ea3a7a50281695 2024-12-03T04:45:14,444 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/bdaa5add4eed4437b7ea3a7a50281695, entries=150, sequenceid=360, filesize=12.0 K 2024-12-03T04:45:14,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/66995955a24e4ce1a7cb90d7550e0d6c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/66995955a24e4ce1a7cb90d7550e0d6c 2024-12-03T04:45:14,448 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/66995955a24e4ce1a7cb90d7550e0d6c, entries=150, sequenceid=360, filesize=12.0 K 2024-12-03T04:45:14,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/28d04c31551c4ad98c8888deb3a596aa as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/28d04c31551c4ad98c8888deb3a596aa 2024-12-03T04:45:14,453 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/28d04c31551c4ad98c8888deb3a596aa, entries=150, sequenceid=360, filesize=12.0 K 2024-12-03T04:45:14,453 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/0bc6345f6c6f4252926799b97563ff07 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/0bc6345f6c6f4252926799b97563ff07 2024-12-03T04:45:14,454 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for a9660218570dfb53504238e06a5637bb in 867ms, sequenceid=360, compaction requested=false 2024-12-03T04:45:14,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:14,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:14,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-03T04:45:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-03T04:45:14,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-03T04:45:14,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-12-03T04:45:14,459 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/C of a9660218570dfb53504238e06a5637bb into 0bc6345f6c6f4252926799b97563ff07(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:14,459 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:14,459 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/C, priority=13, startTime=1733201113583; duration=0sec 2024-12-03T04:45:14,460 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:14,460 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:C 2024-12-03T04:45:14,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.6370 sec 2024-12-03T04:45:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:14,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:45:14,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:14,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:14,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:14,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:14,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:14,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:14,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/b68b4c2b6e9c495580e1b43e851360c0 is 50, key is test_row_0/A:col10/1733201113768/Put/seqid=0 2024-12-03T04:45:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T04:45:14,972 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:14,972 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-03T04:45:14,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T04:45:14,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:14,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:14,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742336_1512 (size=14741) 2024-12-03T04:45:14,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/b68b4c2b6e9c495580e1b43e851360c0 2024-12-03T04:45:15,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/983d8dcc999e4c498284e5b441caf439 is 50, key is test_row_0/B:col10/1733201113768/Put/seqid=0 2024-12-03T04:45:15,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742337_1513 (size=12301) 2024-12-03T04:45:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T04:45:15,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T04:45:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201175119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:15,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201175232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:15,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:15,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:15,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:15,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T04:45:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T04:45:15,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T04:45:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/983d8dcc999e4c498284e5b441caf439 2024-12-03T04:45:15,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/0a3c8376f5764c69932d6428367080f0 is 50, key is test_row_0/C:col10/1733201113768/Put/seqid=0 2024-12-03T04:45:15,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742338_1514 (size=12301) 2024-12-03T04:45:15,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201175441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:15,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:15,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T04:45:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T04:45:15,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T04:45:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:15,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201175762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:15,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/0a3c8376f5764c69932d6428367080f0 2024-12-03T04:45:15,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/b68b4c2b6e9c495580e1b43e851360c0 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b68b4c2b6e9c495580e1b43e851360c0 2024-12-03T04:45:15,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b68b4c2b6e9c495580e1b43e851360c0, entries=200, sequenceid=375, filesize=14.4 K 2024-12-03T04:45:15,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/983d8dcc999e4c498284e5b441caf439 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/983d8dcc999e4c498284e5b441caf439 2024-12-03T04:45:15,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/983d8dcc999e4c498284e5b441caf439, entries=150, sequenceid=375, filesize=12.0 K 2024-12-03T04:45:15,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/0a3c8376f5764c69932d6428367080f0 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/0a3c8376f5764c69932d6428367080f0 2024-12-03T04:45:15,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/0a3c8376f5764c69932d6428367080f0, entries=150, sequenceid=375, filesize=12.0 K 2024-12-03T04:45:15,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a9660218570dfb53504238e06a5637bb in 917ms, sequenceid=375, compaction requested=true 2024-12-03T04:45:15,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:15,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:15,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:15,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:15,865 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:15,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:15,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:15,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:15,865 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:15,866 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:15,866 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:15,866 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/A is initiating minor compaction (all files) 2024-12-03T04:45:15,866 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/B is initiating minor compaction (all files) 2024-12-03T04:45:15,866 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/A in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,866 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/B in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,866 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/1d23552ca34d42a584a2598d4e197236, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/bdaa5add4eed4437b7ea3a7a50281695, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b68b4c2b6e9c495580e1b43e851360c0] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=39.0 K 2024-12-03T04:45:15,866 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/cc9562829a364e4298eb5c7d1d7ceae4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/66995955a24e4ce1a7cb90d7550e0d6c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/983d8dcc999e4c498284e5b441caf439] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=36.4 K 2024-12-03T04:45:15,866 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d23552ca34d42a584a2598d4e197236, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733201111492 2024-12-03T04:45:15,866 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting cc9562829a364e4298eb5c7d1d7ceae4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733201111492 2024-12-03T04:45:15,867 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdaa5add4eed4437b7ea3a7a50281695, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733201112969 2024-12-03T04:45:15,867 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 66995955a24e4ce1a7cb90d7550e0d6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733201112969 2024-12-03T04:45:15,867 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 983d8dcc999e4c498284e5b441caf439, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733201113744 2024-12-03T04:45:15,867 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting b68b4c2b6e9c495580e1b43e851360c0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733201113737 2024-12-03T04:45:15,874 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#B#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:15,874 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#A#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:15,875 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/7a6c1a4ed2c94470bc675b87932359c3 is 50, key is test_row_0/B:col10/1733201113768/Put/seqid=0 2024-12-03T04:45:15,875 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/ddb9a9851e8445a5b34c2fab59170e5b is 50, key is test_row_0/A:col10/1733201113768/Put/seqid=0 2024-12-03T04:45:15,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742339_1515 (size=12813) 2024-12-03T04:45:15,894 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/7a6c1a4ed2c94470bc675b87932359c3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7a6c1a4ed2c94470bc675b87932359c3 2024-12-03T04:45:15,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T04:45:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,897 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:45:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:15,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742340_1516 (size=12949) 2024-12-03T04:45:15,900 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/B of a9660218570dfb53504238e06a5637bb into 7a6c1a4ed2c94470bc675b87932359c3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:15,900 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:15,900 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/B, priority=13, startTime=1733201115865; duration=0sec 2024-12-03T04:45:15,900 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:15,900 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:B 2024-12-03T04:45:15,900 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:15,901 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:15,901 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/C is initiating minor compaction (all files) 2024-12-03T04:45:15,901 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/C in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:15,902 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/0bc6345f6c6f4252926799b97563ff07, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/28d04c31551c4ad98c8888deb3a596aa, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/0a3c8376f5764c69932d6428367080f0] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=36.4 K 2024-12-03T04:45:15,902 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bc6345f6c6f4252926799b97563ff07, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733201111492 2024-12-03T04:45:15,902 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 28d04c31551c4ad98c8888deb3a596aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733201112969 2024-12-03T04:45:15,903 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a3c8376f5764c69932d6428367080f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733201113744 2024-12-03T04:45:15,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/1224d1cf4fdd4dbd8ff6a059932a99f9 is 50, key is test_row_0/A:col10/1733201115117/Put/seqid=0 2024-12-03T04:45:15,911 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#C#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:15,911 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/eea72fa3b0a94413994a5a0e18b08718 is 50, key is test_row_0/C:col10/1733201113768/Put/seqid=0 2024-12-03T04:45:15,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742341_1517 (size=12301) 2024-12-03T04:45:15,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742342_1518 (size=12779) 2024-12-03T04:45:15,931 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/eea72fa3b0a94413994a5a0e18b08718 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/eea72fa3b0a94413994a5a0e18b08718 2024-12-03T04:45:15,938 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/C of a9660218570dfb53504238e06a5637bb into eea72fa3b0a94413994a5a0e18b08718(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:15,938 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:15,938 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/C, priority=13, startTime=1733201115865; duration=0sec 2024-12-03T04:45:15,938 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:15,938 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:C 2024-12-03T04:45:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T04:45:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:16,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:16,319 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/ddb9a9851e8445a5b34c2fab59170e5b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ddb9a9851e8445a5b34c2fab59170e5b 2024-12-03T04:45:16,322 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/1224d1cf4fdd4dbd8ff6a059932a99f9 2024-12-03T04:45:16,330 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/A of a9660218570dfb53504238e06a5637bb into ddb9a9851e8445a5b34c2fab59170e5b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:16,330 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:16,330 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/A, priority=13, startTime=1733201115865; duration=0sec 2024-12-03T04:45:16,331 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:16,331 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:A 2024-12-03T04:45:16,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/2a49a0a3674741869d5b15403b38391e is 50, key is test_row_0/B:col10/1733201115117/Put/seqid=0 2024-12-03T04:45:16,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742343_1519 (size=12301) 2024-12-03T04:45:16,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201176375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:16,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201176492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:16,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201176712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:16,744 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/2a49a0a3674741869d5b15403b38391e 2024-12-03T04:45:16,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/3ff1103344c143b6b885ca10a2151f0e is 50, key is test_row_0/C:col10/1733201115117/Put/seqid=0 2024-12-03T04:45:16,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742344_1520 (size=12301) 2024-12-03T04:45:17,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201177032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T04:45:17,167 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/3ff1103344c143b6b885ca10a2151f0e 2024-12-03T04:45:17,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/1224d1cf4fdd4dbd8ff6a059932a99f9 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/1224d1cf4fdd4dbd8ff6a059932a99f9 2024-12-03T04:45:17,175 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/1224d1cf4fdd4dbd8ff6a059932a99f9, entries=150, sequenceid=395, filesize=12.0 K 2024-12-03T04:45:17,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/2a49a0a3674741869d5b15403b38391e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/2a49a0a3674741869d5b15403b38391e 2024-12-03T04:45:17,179 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/2a49a0a3674741869d5b15403b38391e, entries=150, sequenceid=395, filesize=12.0 K 2024-12-03T04:45:17,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/3ff1103344c143b6b885ca10a2151f0e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ff1103344c143b6b885ca10a2151f0e 2024-12-03T04:45:17,182 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ff1103344c143b6b885ca10a2151f0e, entries=150, sequenceid=395, filesize=12.0 K 2024-12-03T04:45:17,183 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a9660218570dfb53504238e06a5637bb in 1286ms, sequenceid=395, compaction requested=false 2024-12-03T04:45:17,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:17,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:17,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-03T04:45:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-03T04:45:17,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-03T04:45:17,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2090 sec 2024-12-03T04:45:17,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.2150 sec 2024-12-03T04:45:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:17,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T04:45:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:17,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:17,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/7023e2a006d94e81b330a02f228b6e2c is 50, key is test_row_0/A:col10/1733201116371/Put/seqid=0 2024-12-03T04:45:17,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742345_1521 (size=14741) 2024-12-03T04:45:17,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201177671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:17,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201177781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:17,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/7023e2a006d94e81b330a02f228b6e2c 2024-12-03T04:45:17,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/fce5dd374c6245b09890f85cc3bbbd9c is 50, key is test_row_0/B:col10/1733201116371/Put/seqid=0 2024-12-03T04:45:17,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742346_1522 (size=12301) 2024-12-03T04:45:17,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/fce5dd374c6245b09890f85cc3bbbd9c 2024-12-03T04:45:17,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/c38c1421e4db4ff0b015ec6f593f64a3 is 50, key is test_row_0/C:col10/1733201116371/Put/seqid=0 2024-12-03T04:45:17,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742347_1523 (size=12301) 2024-12-03T04:45:17,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/c38c1421e4db4ff0b015ec6f593f64a3 2024-12-03T04:45:17,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/7023e2a006d94e81b330a02f228b6e2c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7023e2a006d94e81b330a02f228b6e2c 2024-12-03T04:45:17,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7023e2a006d94e81b330a02f228b6e2c, entries=200, sequenceid=411, filesize=14.4 K 2024-12-03T04:45:17,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/fce5dd374c6245b09890f85cc3bbbd9c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fce5dd374c6245b09890f85cc3bbbd9c 2024-12-03T04:45:18,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fce5dd374c6245b09890f85cc3bbbd9c, entries=150, sequenceid=411, filesize=12.0 K 2024-12-03T04:45:18,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/c38c1421e4db4ff0b015ec6f593f64a3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c38c1421e4db4ff0b015ec6f593f64a3 2024-12-03T04:45:18,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c38c1421e4db4ff0b015ec6f593f64a3, entries=150, sequenceid=411, filesize=12.0 K 2024-12-03T04:45:18,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a9660218570dfb53504238e06a5637bb in 449ms, sequenceid=411, compaction requested=true 2024-12-03T04:45:18,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:18,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:18,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:18,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:18,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:18,006 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:18,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:18,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:18,006 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:18,007 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:18,007 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:18,007 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/B is initiating minor compaction (all files) 2024-12-03T04:45:18,007 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/A is initiating minor compaction (all files) 2024-12-03T04:45:18,007 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/B in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:18,007 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/A in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:18,007 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7a6c1a4ed2c94470bc675b87932359c3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/2a49a0a3674741869d5b15403b38391e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fce5dd374c6245b09890f85cc3bbbd9c] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=36.5 K 2024-12-03T04:45:18,007 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ddb9a9851e8445a5b34c2fab59170e5b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/1224d1cf4fdd4dbd8ff6a059932a99f9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7023e2a006d94e81b330a02f228b6e2c] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=39.1 K 2024-12-03T04:45:18,008 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddb9a9851e8445a5b34c2fab59170e5b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733201113744 2024-12-03T04:45:18,008 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a6c1a4ed2c94470bc675b87932359c3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733201113744 2024-12-03T04:45:18,008 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1224d1cf4fdd4dbd8ff6a059932a99f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733201115027 2024-12-03T04:45:18,008 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a49a0a3674741869d5b15403b38391e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733201115027 2024-12-03T04:45:18,008 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting fce5dd374c6245b09890f85cc3bbbd9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733201116287 2024-12-03T04:45:18,008 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7023e2a006d94e81b330a02f228b6e2c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733201116287 2024-12-03T04:45:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:18,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:45:18,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:18,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:18,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:18,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:18,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:18,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:18,017 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#A#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:18,018 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/26798bb30c9549eab082b6c255306bb3 is 50, key is test_row_0/A:col10/1733201116371/Put/seqid=0 2024-12-03T04:45:18,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/3befbdf3808248eba7a7d01c7f031086 is 50, key is test_row_0/A:col10/1733201118010/Put/seqid=0 2024-12-03T04:45:18,030 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:18,030 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/1ff868be12684465b88e5a24b7e52b7e is 50, key is test_row_0/B:col10/1733201116371/Put/seqid=0 2024-12-03T04:45:18,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742350_1526 (size=12915) 2024-12-03T04:45:18,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742348_1524 (size=13051) 2024-12-03T04:45:18,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742349_1525 (size=14741) 2024-12-03T04:45:18,052 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/1ff868be12684465b88e5a24b7e52b7e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/1ff868be12684465b88e5a24b7e52b7e 2024-12-03T04:45:18,055 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/B of a9660218570dfb53504238e06a5637bb into 1ff868be12684465b88e5a24b7e52b7e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:18,056 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:18,056 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/B, priority=13, startTime=1733201118006; duration=0sec 2024-12-03T04:45:18,056 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:18,056 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:B 2024-12-03T04:45:18,056 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:18,056 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:18,057 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/C is initiating minor compaction (all files) 2024-12-03T04:45:18,057 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/C in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:18,057 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/eea72fa3b0a94413994a5a0e18b08718, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ff1103344c143b6b885ca10a2151f0e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c38c1421e4db4ff0b015ec6f593f64a3] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=36.5 K 2024-12-03T04:45:18,057 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting eea72fa3b0a94413994a5a0e18b08718, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733201113744 2024-12-03T04:45:18,057 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ff1103344c143b6b885ca10a2151f0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733201115027 2024-12-03T04:45:18,058 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c38c1421e4db4ff0b015ec6f593f64a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733201116287 2024-12-03T04:45:18,064 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:18,065 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/cd43bf0b6cd94b568ec05555f3e77a5e is 50, key is test_row_0/C:col10/1733201116371/Put/seqid=0 2024-12-03T04:45:18,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742351_1527 (size=12881) 2024-12-03T04:45:18,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 331 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201178095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:18,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201178211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:18,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201178431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:18,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/3befbdf3808248eba7a7d01c7f031086 2024-12-03T04:45:18,495 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/26798bb30c9549eab082b6c255306bb3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/26798bb30c9549eab082b6c255306bb3 2024-12-03T04:45:18,495 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/cd43bf0b6cd94b568ec05555f3e77a5e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/cd43bf0b6cd94b568ec05555f3e77a5e 2024-12-03T04:45:18,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/979f3cd6980a4dfb8a157eb48c0ac9ab is 50, key is test_row_0/B:col10/1733201118010/Put/seqid=0 2024-12-03T04:45:18,511 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/A of a9660218570dfb53504238e06a5637bb into 26798bb30c9549eab082b6c255306bb3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:18,511 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:18,511 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/A, priority=13, startTime=1733201118006; duration=0sec 2024-12-03T04:45:18,512 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:18,512 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:A 2024-12-03T04:45:18,514 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/C of a9660218570dfb53504238e06a5637bb into cd43bf0b6cd94b568ec05555f3e77a5e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:18,514 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:18,514 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/C, priority=13, startTime=1733201118006; duration=0sec 2024-12-03T04:45:18,514 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:18,514 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:C 2024-12-03T04:45:18,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742352_1528 (size=12301) 2024-12-03T04:45:18,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/979f3cd6980a4dfb8a157eb48c0ac9ab 2024-12-03T04:45:18,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/1a7b5849f91a4d80947535c0255d5f1c is 50, key is test_row_0/C:col10/1733201118010/Put/seqid=0 2024-12-03T04:45:18,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742353_1529 (size=12301) 2024-12-03T04:45:18,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/1a7b5849f91a4d80947535c0255d5f1c 2024-12-03T04:45:18,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/3befbdf3808248eba7a7d01c7f031086 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3befbdf3808248eba7a7d01c7f031086 2024-12-03T04:45:18,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3befbdf3808248eba7a7d01c7f031086, entries=200, sequenceid=431, filesize=14.4 K 2024-12-03T04:45:18,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/979f3cd6980a4dfb8a157eb48c0ac9ab as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/979f3cd6980a4dfb8a157eb48c0ac9ab 2024-12-03T04:45:18,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/979f3cd6980a4dfb8a157eb48c0ac9ab, entries=150, sequenceid=431, filesize=12.0 K 2024-12-03T04:45:18,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/1a7b5849f91a4d80947535c0255d5f1c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1a7b5849f91a4d80947535c0255d5f1c 2024-12-03T04:45:18,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1a7b5849f91a4d80947535c0255d5f1c, entries=150, sequenceid=431, filesize=12.0 K 2024-12-03T04:45:18,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a9660218570dfb53504238e06a5637bb in 614ms, sequenceid=431, compaction requested=false 2024-12-03T04:45:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:18,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T04:45:18,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:18,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:18,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:18,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:18,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:18,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:18,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/89b61e8f1bfa4a708966f38cccfbde28 is 50, key is test_row_0/A:col10/1733201118753/Put/seqid=0 2024-12-03T04:45:18,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742354_1530 (size=14741) 2024-12-03T04:45:18,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/89b61e8f1bfa4a708966f38cccfbde28 2024-12-03T04:45:18,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/fb922055487d4a7fb97cb4227decca51 is 50, key is test_row_0/B:col10/1733201118753/Put/seqid=0 2024-12-03T04:45:18,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742355_1531 (size=12301) 2024-12-03T04:45:18,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201178930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:19,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 352 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201179053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T04:45:19,139 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:19,144 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-03T04:45:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T04:45:19,153 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:19,156 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:19,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:19,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/fb922055487d4a7fb97cb4227decca51 2024-12-03T04:45:19,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/fc089f41899f405a8820313b5a8ab03e is 50, key is test_row_0/C:col10/1733201118753/Put/seqid=0 2024-12-03T04:45:19,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742356_1532 (size=12301) 2024-12-03T04:45:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T04:45:19,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201179281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:19,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T04:45:19,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:19,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:19,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:19,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:19,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:19,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T04:45:19,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:19,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:19,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:19,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:19,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T04:45:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 354 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201179594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:19,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T04:45:19,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:19,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:19,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:19,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:19,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:19,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/fc089f41899f405a8820313b5a8ab03e 2024-12-03T04:45:19,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/89b61e8f1bfa4a708966f38cccfbde28 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/89b61e8f1bfa4a708966f38cccfbde28 2024-12-03T04:45:19,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/89b61e8f1bfa4a708966f38cccfbde28, entries=200, sequenceid=447, filesize=14.4 K 2024-12-03T04:45:19,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/fb922055487d4a7fb97cb4227decca51 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fb922055487d4a7fb97cb4227decca51 2024-12-03T04:45:19,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fb922055487d4a7fb97cb4227decca51, entries=150, sequenceid=447, filesize=12.0 K 2024-12-03T04:45:19,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/fc089f41899f405a8820313b5a8ab03e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/fc089f41899f405a8820313b5a8ab03e 2024-12-03T04:45:19,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/fc089f41899f405a8820313b5a8ab03e, entries=150, sequenceid=447, filesize=12.0 K 2024-12-03T04:45:19,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a9660218570dfb53504238e06a5637bb in 953ms, sequenceid=447, compaction requested=true 2024-12-03T04:45:19,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:19,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:19,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:19,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:19,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T04:45:19,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:19,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T04:45:19,709 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:19,709 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:19,717 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:19,717 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/A is initiating minor compaction (all files) 2024-12-03T04:45:19,717 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/A in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:19,717 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/26798bb30c9549eab082b6c255306bb3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3befbdf3808248eba7a7d01c7f031086, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/89b61e8f1bfa4a708966f38cccfbde28] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=41.5 K 2024-12-03T04:45:19,720 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:19,720 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 26798bb30c9549eab082b6c255306bb3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733201116287 2024-12-03T04:45:19,720 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/C is initiating minor compaction (all files) 2024-12-03T04:45:19,720 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/C in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:19,720 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/cd43bf0b6cd94b568ec05555f3e77a5e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1a7b5849f91a4d80947535c0255d5f1c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/fc089f41899f405a8820313b5a8ab03e] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=36.6 K 2024-12-03T04:45:19,721 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd43bf0b6cd94b568ec05555f3e77a5e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733201116287 2024-12-03T04:45:19,721 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3befbdf3808248eba7a7d01c7f031086, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733201117639 2024-12-03T04:45:19,722 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a7b5849f91a4d80947535c0255d5f1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733201117639 2024-12-03T04:45:19,722 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 89b61e8f1bfa4a708966f38cccfbde28, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733201118041 2024-12-03T04:45:19,723 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc089f41899f405a8820313b5a8ab03e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733201118041 2024-12-03T04:45:19,744 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#C#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:19,744 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/10c194a32cae4a049f24fbf1fa011bd9 is 50, key is test_row_0/C:col10/1733201118753/Put/seqid=0 2024-12-03T04:45:19,758 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#A#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:19,759 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/cef90494769e48de9eb20d6f530f2bcf is 50, key is test_row_0/A:col10/1733201118753/Put/seqid=0 2024-12-03T04:45:19,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742358_1534 (size=13153) 2024-12-03T04:45:19,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742357_1533 (size=12983) 2024-12-03T04:45:19,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T04:45:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:19,777 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T04:45:19,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:19,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:19,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:19,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:19,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:19,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T04:45:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/8bc61e448ce44f94a5bfa66169fb2596 is 50, key is test_row_0/A:col10/1733201118917/Put/seqid=0 2024-12-03T04:45:19,804 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/10c194a32cae4a049f24fbf1fa011bd9 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/10c194a32cae4a049f24fbf1fa011bd9 2024-12-03T04:45:19,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742359_1535 (size=12301) 2024-12-03T04:45:19,812 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/8bc61e448ce44f94a5bfa66169fb2596 2024-12-03T04:45:19,813 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/C of a9660218570dfb53504238e06a5637bb into 10c194a32cae4a049f24fbf1fa011bd9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:19,814 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:19,814 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/C, priority=13, startTime=1733201119708; duration=0sec 2024-12-03T04:45:19,814 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:19,814 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:C 2024-12-03T04:45:19,814 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:19,816 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:19,816 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/B is initiating minor compaction (all files) 2024-12-03T04:45:19,816 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/B in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:19,816 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/1ff868be12684465b88e5a24b7e52b7e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/979f3cd6980a4dfb8a157eb48c0ac9ab, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fb922055487d4a7fb97cb4227decca51] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=36.6 K 2024-12-03T04:45:19,821 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ff868be12684465b88e5a24b7e52b7e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733201116287 2024-12-03T04:45:19,826 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 979f3cd6980a4dfb8a157eb48c0ac9ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733201117639 2024-12-03T04:45:19,827 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb922055487d4a7fb97cb4227decca51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733201118041 2024-12-03T04:45:19,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/da8fa1a55f8847c5958a9de531884af4 is 50, key is test_row_0/B:col10/1733201118917/Put/seqid=0 2024-12-03T04:45:19,841 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#B#compaction#461 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:19,841 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/3dbeea5d5b2e47158fe10215362c7ae9 is 50, key is test_row_0/B:col10/1733201118753/Put/seqid=0 2024-12-03T04:45:19,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742360_1536 (size=12301) 2024-12-03T04:45:19,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742361_1537 (size=13017) 2024-12-03T04:45:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:20,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:20,176 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/cef90494769e48de9eb20d6f530f2bcf as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/cef90494769e48de9eb20d6f530f2bcf 2024-12-03T04:45:20,181 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/A of a9660218570dfb53504238e06a5637bb into cef90494769e48de9eb20d6f530f2bcf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:20,181 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:20,181 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/A, priority=13, startTime=1733201119708; duration=0sec 2024-12-03T04:45:20,181 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:20,181 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:A 2024-12-03T04:45:20,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 365 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201180207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:20,251 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/da8fa1a55f8847c5958a9de531884af4 2024-12-03T04:45:20,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/52122df04a934fee8684dfbe7f72ba0d is 50, key is test_row_0/C:col10/1733201118917/Put/seqid=0 2024-12-03T04:45:20,275 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/3dbeea5d5b2e47158fe10215362c7ae9 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/3dbeea5d5b2e47158fe10215362c7ae9 2024-12-03T04:45:20,287 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/B of a9660218570dfb53504238e06a5637bb into 3dbeea5d5b2e47158fe10215362c7ae9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:20,287 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:20,287 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/B, priority=13, startTime=1733201119708; duration=0sec 2024-12-03T04:45:20,287 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:20,287 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:B 2024-12-03T04:45:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T04:45:20,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742362_1538 (size=12301) 2024-12-03T04:45:20,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 366 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201180321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:20,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:20,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:20,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 367 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201180531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:20,701 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/52122df04a934fee8684dfbe7f72ba0d 2024-12-03T04:45:20,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/8bc61e448ce44f94a5bfa66169fb2596 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8bc61e448ce44f94a5bfa66169fb2596 2024-12-03T04:45:20,707 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8bc61e448ce44f94a5bfa66169fb2596, entries=150, sequenceid=466, filesize=12.0 K 2024-12-03T04:45:20,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/da8fa1a55f8847c5958a9de531884af4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/da8fa1a55f8847c5958a9de531884af4 2024-12-03T04:45:20,711 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/da8fa1a55f8847c5958a9de531884af4, entries=150, sequenceid=466, filesize=12.0 K 2024-12-03T04:45:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/52122df04a934fee8684dfbe7f72ba0d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/52122df04a934fee8684dfbe7f72ba0d 2024-12-03T04:45:20,715 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/52122df04a934fee8684dfbe7f72ba0d, entries=150, sequenceid=466, filesize=12.0 K 2024-12-03T04:45:20,716 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a9660218570dfb53504238e06a5637bb in 939ms, sequenceid=466, compaction requested=false 2024-12-03T04:45:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-03T04:45:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-03T04:45:20,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-03T04:45:20,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5610 sec 2024-12-03T04:45:20,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5750 sec 2024-12-03T04:45:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:20,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T04:45:20,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:20,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:20,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:20,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:20,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:20,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:20,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/3833c737588d43288983a02522353022 is 50, key is test_row_0/A:col10/1733201120125/Put/seqid=0 2024-12-03T04:45:20,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742363_1539 (size=14741) 2024-12-03T04:45:21,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 384 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201180991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:21,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 385 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201181112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:21,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/3833c737588d43288983a02522353022 2024-12-03T04:45:21,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/ea92346f85444c3a8d2f6d5681381830 is 50, key is test_row_0/B:col10/1733201120125/Put/seqid=0 2024-12-03T04:45:21,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742364_1540 (size=12301) 2024-12-03T04:45:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T04:45:21,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/ea92346f85444c3a8d2f6d5681381830 2024-12-03T04:45:21,302 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:21,302 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-03T04:45:21,305 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:21,307 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:21,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T04:45:21,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/d277e431bc184c48a0ba1b1506de01f7 is 50, key is test_row_0/C:col10/1733201120125/Put/seqid=0 2024-12-03T04:45:21,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742365_1541 (size=12301) 2024-12-03T04:45:21,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 386 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201181332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T04:45:21,433 INFO [Thread-1991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1991. 2024-12-03T04:45:21,434 DEBUG [Thread-1991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:21,434 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,434 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,434 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:21,434 INFO [Thread-1987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1987. 2024-12-03T04:45:21,434 DEBUG [Thread-1987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:21,434 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,434 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:21,436 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-12-03T04:45:21,436 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:21,436 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,436 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:21,446 INFO [Thread-1989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1989. 2024-12-03T04:45:21,446 DEBUG [Thread-1989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:21,446 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,446 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:21,446 INFO [Thread-1985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1985. 2024-12-03T04:45:21,446 DEBUG [Thread-1985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:21,446 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,446 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:21,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T04:45:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:21,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:21,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T04:45:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:21,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T04:45:21,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 387 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1733201181652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 , the old value is region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., hostname=32beccd76b35,43193,1733200986730, seqNum=2 because the exception is null or not the one we care about 2024-12-03T04:45:21,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/d277e431bc184c48a0ba1b1506de01f7 2024-12-03T04:45:21,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/3833c737588d43288983a02522353022 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3833c737588d43288983a02522353022 2024-12-03T04:45:21,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3833c737588d43288983a02522353022, entries=200, sequenceid=483, filesize=14.4 K 2024-12-03T04:45:21,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/ea92346f85444c3a8d2f6d5681381830 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/ea92346f85444c3a8d2f6d5681381830 2024-12-03T04:45:21,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/ea92346f85444c3a8d2f6d5681381830, entries=150, sequenceid=483, filesize=12.0 K 2024-12-03T04:45:21,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/d277e431bc184c48a0ba1b1506de01f7 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/d277e431bc184c48a0ba1b1506de01f7 2024-12-03T04:45:21,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/d277e431bc184c48a0ba1b1506de01f7, entries=150, sequenceid=483, filesize=12.0 K 2024-12-03T04:45:21,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a9660218570dfb53504238e06a5637bb in 878ms, sequenceid=483, compaction requested=true 2024-12-03T04:45:21,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:21,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:21,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:21,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:21,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:21,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9660218570dfb53504238e06a5637bb:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:21,737 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:21,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:21,737 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:21,737 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:21,737 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:21,737 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/B is initiating minor compaction (all files) 2024-12-03T04:45:21,737 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/A is initiating minor compaction (all files) 2024-12-03T04:45:21,738 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/B in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:21,738 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/A in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:21,738 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/3dbeea5d5b2e47158fe10215362c7ae9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/da8fa1a55f8847c5958a9de531884af4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/ea92346f85444c3a8d2f6d5681381830] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=36.7 K 2024-12-03T04:45:21,738 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/cef90494769e48de9eb20d6f530f2bcf, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8bc61e448ce44f94a5bfa66169fb2596, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3833c737588d43288983a02522353022] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=39.3 K 2024-12-03T04:45:21,738 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dbeea5d5b2e47158fe10215362c7ae9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733201118041 2024-12-03T04:45:21,738 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef90494769e48de9eb20d6f530f2bcf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733201118041 2024-12-03T04:45:21,738 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting da8fa1a55f8847c5958a9de531884af4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733201118891 2024-12-03T04:45:21,738 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bc61e448ce44f94a5bfa66169fb2596, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733201118891 2024-12-03T04:45:21,738 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting ea92346f85444c3a8d2f6d5681381830, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733201120125 2024-12-03T04:45:21,738 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3833c737588d43288983a02522353022, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733201120125 2024-12-03T04:45:21,744 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#A#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:21,745 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/448335e6770a40b9a8cf265dbc4d4536 is 50, key is test_row_0/A:col10/1733201120125/Put/seqid=0 2024-12-03T04:45:21,745 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#B#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:21,745 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/bb428a11befd465dac180ad69f4a60f4 is 50, key is test_row_0/B:col10/1733201120125/Put/seqid=0 2024-12-03T04:45:21,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742366_1542 (size=13255) 2024-12-03T04:45:21,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742367_1543 (size=13119) 2024-12-03T04:45:21,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T04:45:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:21,766 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T04:45:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:21,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/0067edee795943dbb1561bc5003fefbc is 50, key is test_row_0/A:col10/1733201120972/Put/seqid=0 2024-12-03T04:45:21,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742368_1544 (size=12301) 2024-12-03T04:45:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. as already flushing 2024-12-03T04:45:21,813 INFO [Thread-1982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1982. 2024-12-03T04:45:21,813 DEBUG [Thread-1982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:21,813 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,813 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,813 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:21,823 INFO [Thread-1974 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1974. 2024-12-03T04:45:21,823 DEBUG [Thread-1974 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:21,823 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,823 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:21,843 INFO [Thread-1978 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1978. 2024-12-03T04:45:21,843 DEBUG [Thread-1978 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:21,843 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,844 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:21,852 INFO [Thread-1980 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1980. 2024-12-03T04:45:21,853 DEBUG [Thread-1980 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:21,853 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,853 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:21,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T04:45:22,157 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/bb428a11befd465dac180ad69f4a60f4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/bb428a11befd465dac180ad69f4a60f4 2024-12-03T04:45:22,159 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/448335e6770a40b9a8cf265dbc4d4536 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/448335e6770a40b9a8cf265dbc4d4536 2024-12-03T04:45:22,161 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/B of a9660218570dfb53504238e06a5637bb into bb428a11befd465dac180ad69f4a60f4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:22,161 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:22,162 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/B, priority=13, startTime=1733201121737; duration=0sec 2024-12-03T04:45:22,162 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:22,162 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:B 2024-12-03T04:45:22,162 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:22,163 INFO [Thread-1976 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1976. 2024-12-03T04:45:22,163 DEBUG [Thread-1976 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:22,163 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:22,163 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:22,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:22,165 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:22,165 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): a9660218570dfb53504238e06a5637bb/C is initiating minor compaction (all files) 2024-12-03T04:45:22,165 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9660218570dfb53504238e06a5637bb/C in TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:22,165 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/10c194a32cae4a049f24fbf1fa011bd9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/52122df04a934fee8684dfbe7f72ba0d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/d277e431bc184c48a0ba1b1506de01f7] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp, totalSize=36.7 K 2024-12-03T04:45:22,165 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/A of a9660218570dfb53504238e06a5637bb into 448335e6770a40b9a8cf265dbc4d4536(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:22,165 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:22,165 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/A, priority=13, startTime=1733201121736; duration=0sec 2024-12-03T04:45:22,165 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:22,165 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 10c194a32cae4a049f24fbf1fa011bd9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1733201118041 2024-12-03T04:45:22,165 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:A 2024-12-03T04:45:22,166 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 52122df04a934fee8684dfbe7f72ba0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733201118891 2024-12-03T04:45:22,166 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting d277e431bc184c48a0ba1b1506de01f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733201120125 2024-12-03T04:45:22,171 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9660218570dfb53504238e06a5637bb#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:22,172 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/ce263b1cf5024f0ba1f4cac3243886be is 50, key is test_row_0/C:col10/1733201120125/Put/seqid=0 2024-12-03T04:45:22,172 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/0067edee795943dbb1561bc5003fefbc 2024-12-03T04:45:22,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742369_1545 (size=13085) 2024-12-03T04:45:22,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/905a655b673f4ef3a392c2862998dcde is 50, key is test_row_0/B:col10/1733201120972/Put/seqid=0 2024-12-03T04:45:22,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742370_1546 (size=12301) 2024-12-03T04:45:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T04:45:22,582 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/905a655b673f4ef3a392c2862998dcde 2024-12-03T04:45:22,583 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/ce263b1cf5024f0ba1f4cac3243886be as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/ce263b1cf5024f0ba1f4cac3243886be 2024-12-03T04:45:22,586 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9660218570dfb53504238e06a5637bb/C of a9660218570dfb53504238e06a5637bb into ce263b1cf5024f0ba1f4cac3243886be(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:22,586 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:22,586 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., storeName=a9660218570dfb53504238e06a5637bb/C, priority=13, startTime=1733201121737; duration=0sec 2024-12-03T04:45:22,586 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:22,586 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9660218570dfb53504238e06a5637bb:C 2024-12-03T04:45:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/ca65e2f38fac40e9a6b96822fdf752c5 is 50, key is test_row_0/C:col10/1733201120972/Put/seqid=0 2024-12-03T04:45:22,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742371_1547 (size=12301) 2024-12-03T04:45:22,991 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/ca65e2f38fac40e9a6b96822fdf752c5 2024-12-03T04:45:22,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/0067edee795943dbb1561bc5003fefbc as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/0067edee795943dbb1561bc5003fefbc 2024-12-03T04:45:22,996 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/0067edee795943dbb1561bc5003fefbc, entries=150, sequenceid=501, filesize=12.0 K 2024-12-03T04:45:22,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/905a655b673f4ef3a392c2862998dcde as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/905a655b673f4ef3a392c2862998dcde 2024-12-03T04:45:22,999 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/905a655b673f4ef3a392c2862998dcde, entries=150, sequenceid=501, filesize=12.0 K 2024-12-03T04:45:23,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/ca65e2f38fac40e9a6b96822fdf752c5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/ca65e2f38fac40e9a6b96822fdf752c5 2024-12-03T04:45:23,003 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/ca65e2f38fac40e9a6b96822fdf752c5, entries=150, sequenceid=501, filesize=12.0 K 2024-12-03T04:45:23,003 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=33.54 KB/34350 for a9660218570dfb53504238e06a5637bb in 1237ms, sequenceid=501, compaction requested=false 2024-12-03T04:45:23,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for a9660218570dfb53504238e06a5637bb: 2024-12-03T04:45:23,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:23,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-03T04:45:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-03T04:45:23,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-03T04:45:23,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6980 sec 2024-12-03T04:45:23,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7040 sec 2024-12-03T04:45:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T04:45:23,452 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 325 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 955 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2864 rows 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 962 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2886 rows 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 967 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2899 rows 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 950 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2847 rows 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 961 2024-12-03T04:45:23,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2883 rows 2024-12-03T04:45:23,452 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T04:45:23,452 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T04:45:23,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:23,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:23,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:23,453 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T04:45:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T04:45:23,455 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201123455"}]},"ts":"1733201123455"} 2024-12-03T04:45:23,456 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T04:45:23,457 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T04:45:23,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T04:45:23,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9660218570dfb53504238e06a5637bb, UNASSIGN}] 2024-12-03T04:45:23,459 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9660218570dfb53504238e06a5637bb, UNASSIGN 2024-12-03T04:45:23,459 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=a9660218570dfb53504238e06a5637bb, regionState=CLOSING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:45:23,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9660218570dfb53504238e06a5637bb, UNASSIGN because future has completed 2024-12-03T04:45:23,461 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T04:45:23,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:45:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T04:45:23,612 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:23,612 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T04:45:23,613 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing a9660218570dfb53504238e06a5637bb, disabling compactions & flushes 2024-12-03T04:45:23,613 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:23,613 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:23,613 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. after waiting 0 ms 2024-12-03T04:45:23,613 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:23,613 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(2902): Flushing a9660218570dfb53504238e06a5637bb 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T04:45:23,613 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=A 2024-12-03T04:45:23,613 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:23,613 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=B 2024-12-03T04:45:23,613 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:23,613 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a9660218570dfb53504238e06a5637bb, store=C 2024-12-03T04:45:23,613 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:23,616 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/e310a185a62f444cb4d33994c38f88f4 is 50, key is test_row_0/A:col10/1733201121851/Put/seqid=0 2024-12-03T04:45:23,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742372_1548 (size=12301) 2024-12-03T04:45:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T04:45:24,020 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/e310a185a62f444cb4d33994c38f88f4 2024-12-03T04:45:24,025 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/82a936ce61a841fdb7a49a26eb3208d6 is 50, key is test_row_0/B:col10/1733201121851/Put/seqid=0 2024-12-03T04:45:24,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742373_1549 (size=12301) 2024-12-03T04:45:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T04:45:24,429 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/82a936ce61a841fdb7a49a26eb3208d6 2024-12-03T04:45:24,437 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/011ae5f1db29480c8c1a0634f228d2d0 is 50, key is test_row_0/C:col10/1733201121851/Put/seqid=0 2024-12-03T04:45:24,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742374_1550 (size=12301) 2024-12-03T04:45:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T04:45:24,847 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/011ae5f1db29480c8c1a0634f228d2d0 2024-12-03T04:45:24,853 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/A/e310a185a62f444cb4d33994c38f88f4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e310a185a62f444cb4d33994c38f88f4 2024-12-03T04:45:24,857 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e310a185a62f444cb4d33994c38f88f4, entries=150, sequenceid=512, filesize=12.0 K 2024-12-03T04:45:24,858 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/B/82a936ce61a841fdb7a49a26eb3208d6 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/82a936ce61a841fdb7a49a26eb3208d6 2024-12-03T04:45:24,862 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/82a936ce61a841fdb7a49a26eb3208d6, entries=150, sequenceid=512, filesize=12.0 K 2024-12-03T04:45:24,863 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/.tmp/C/011ae5f1db29480c8c1a0634f228d2d0 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/011ae5f1db29480c8c1a0634f228d2d0 2024-12-03T04:45:24,866 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/011ae5f1db29480c8c1a0634f228d2d0, entries=150, sequenceid=512, filesize=12.0 K 2024-12-03T04:45:24,867 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a9660218570dfb53504238e06a5637bb in 1253ms, sequenceid=512, compaction requested=true 2024-12-03T04:45:24,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ce2b54c991164e549547f70ee5e3dbbd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/4bfd5b7580bf4d42a6e9d7273f321602, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/86716c60453f468eac4a12f0fe1e77dc, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f6b867bc8e91497fb5a3010af0b490b1, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/05c5ccf058ba4fd482facdf0b0f39f70, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7682ab35da774cfd8c1ddd5c28667619, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f7d4a6cd1d3640c58aeea6224b651276, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/148b9d51180d4d8594719a5c018c703c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f21d597a973740e295a91d64c5c047e2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e2ea407d79c542a2a8f6e9cf9f660e84, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/90d02793deb049c383fc1b99cd92a64f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/aa551881c77f47f99bc2de529bf4493b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7f1c4d50ba8440bd9e5e3296114f887f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ea013109c33d4ffeab362f07cc323608, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8c71a0b6907f47ad8dae7a56c116023a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e68be19b1cf44eba98ab1a90c79c30cc, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/1d23552ca34d42a584a2598d4e197236, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/bdaa5add4eed4437b7ea3a7a50281695, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b68b4c2b6e9c495580e1b43e851360c0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ddb9a9851e8445a5b34c2fab59170e5b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/1224d1cf4fdd4dbd8ff6a059932a99f9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7023e2a006d94e81b330a02f228b6e2c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/26798bb30c9549eab082b6c255306bb3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3befbdf3808248eba7a7d01c7f031086, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/89b61e8f1bfa4a708966f38cccfbde28, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/cef90494769e48de9eb20d6f530f2bcf, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8bc61e448ce44f94a5bfa66169fb2596, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3833c737588d43288983a02522353022] to archive 2024-12-03T04:45:24,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:45:24,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ce2b54c991164e549547f70ee5e3dbbd to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ce2b54c991164e549547f70ee5e3dbbd 2024-12-03T04:45:24,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/4bfd5b7580bf4d42a6e9d7273f321602 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/4bfd5b7580bf4d42a6e9d7273f321602 2024-12-03T04:45:24,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/86716c60453f468eac4a12f0fe1e77dc to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/86716c60453f468eac4a12f0fe1e77dc 2024-12-03T04:45:24,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f6b867bc8e91497fb5a3010af0b490b1 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f6b867bc8e91497fb5a3010af0b490b1 2024-12-03T04:45:24,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/05c5ccf058ba4fd482facdf0b0f39f70 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/05c5ccf058ba4fd482facdf0b0f39f70 2024-12-03T04:45:24,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7682ab35da774cfd8c1ddd5c28667619 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7682ab35da774cfd8c1ddd5c28667619 2024-12-03T04:45:24,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f7d4a6cd1d3640c58aeea6224b651276 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f7d4a6cd1d3640c58aeea6224b651276 2024-12-03T04:45:24,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/148b9d51180d4d8594719a5c018c703c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/148b9d51180d4d8594719a5c018c703c 2024-12-03T04:45:24,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f21d597a973740e295a91d64c5c047e2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/f21d597a973740e295a91d64c5c047e2 2024-12-03T04:45:24,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e2ea407d79c542a2a8f6e9cf9f660e84 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e2ea407d79c542a2a8f6e9cf9f660e84 2024-12-03T04:45:24,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/90d02793deb049c383fc1b99cd92a64f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/90d02793deb049c383fc1b99cd92a64f 2024-12-03T04:45:24,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/aa551881c77f47f99bc2de529bf4493b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/aa551881c77f47f99bc2de529bf4493b 2024-12-03T04:45:24,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7f1c4d50ba8440bd9e5e3296114f887f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7f1c4d50ba8440bd9e5e3296114f887f 2024-12-03T04:45:24,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ea013109c33d4ffeab362f07cc323608 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ea013109c33d4ffeab362f07cc323608 2024-12-03T04:45:24,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8c71a0b6907f47ad8dae7a56c116023a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8c71a0b6907f47ad8dae7a56c116023a 2024-12-03T04:45:24,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e68be19b1cf44eba98ab1a90c79c30cc to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e68be19b1cf44eba98ab1a90c79c30cc 2024-12-03T04:45:24,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/1d23552ca34d42a584a2598d4e197236 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/1d23552ca34d42a584a2598d4e197236 2024-12-03T04:45:24,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/bdaa5add4eed4437b7ea3a7a50281695 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/bdaa5add4eed4437b7ea3a7a50281695 2024-12-03T04:45:24,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b68b4c2b6e9c495580e1b43e851360c0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/b68b4c2b6e9c495580e1b43e851360c0 2024-12-03T04:45:24,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ddb9a9851e8445a5b34c2fab59170e5b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/ddb9a9851e8445a5b34c2fab59170e5b 2024-12-03T04:45:24,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/1224d1cf4fdd4dbd8ff6a059932a99f9 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/1224d1cf4fdd4dbd8ff6a059932a99f9 2024-12-03T04:45:24,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7023e2a006d94e81b330a02f228b6e2c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/7023e2a006d94e81b330a02f228b6e2c 2024-12-03T04:45:24,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/26798bb30c9549eab082b6c255306bb3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/26798bb30c9549eab082b6c255306bb3 2024-12-03T04:45:24,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3befbdf3808248eba7a7d01c7f031086 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3befbdf3808248eba7a7d01c7f031086 2024-12-03T04:45:24,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/89b61e8f1bfa4a708966f38cccfbde28 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/89b61e8f1bfa4a708966f38cccfbde28 2024-12-03T04:45:24,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/cef90494769e48de9eb20d6f530f2bcf to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/cef90494769e48de9eb20d6f530f2bcf 2024-12-03T04:45:24,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8bc61e448ce44f94a5bfa66169fb2596 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/8bc61e448ce44f94a5bfa66169fb2596 2024-12-03T04:45:24,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3833c737588d43288983a02522353022 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/3833c737588d43288983a02522353022 2024-12-03T04:45:24,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/0fa1b9b2638e4dc781d241744a13ed00, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7d16c770c9204df183962baff52df763, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/5ae4ea290c9e458c996ea959c6f990e2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/abe513bbed4148e199aae94f378cb1da, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/b34b2952b5a04b769be74ea300a9727f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7e95852ed9f74f3facb8c245f61302df, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/99d6f41cda7a49e49b96f16d357c4bef, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/18a67a99d1e949d3a70cd1c8438d5a4f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/1e3264fa8d7e47aeaa2b9d2fc63c6b1b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c44e84b997b0496f9aadef25fbbc2d86, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/274420c727394b9198b386e51dcd1f70, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/cc9562829a364e4298eb5c7d1d7ceae4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c615d7e3e73c4574a9c91e11349f4e92, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/66995955a24e4ce1a7cb90d7550e0d6c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7a6c1a4ed2c94470bc675b87932359c3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/983d8dcc999e4c498284e5b441caf439, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/2a49a0a3674741869d5b15403b38391e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/1ff868be12684465b88e5a24b7e52b7e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fce5dd374c6245b09890f85cc3bbbd9c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/979f3cd6980a4dfb8a157eb48c0ac9ab, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/3dbeea5d5b2e47158fe10215362c7ae9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fb922055487d4a7fb97cb4227decca51, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/da8fa1a55f8847c5958a9de531884af4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/ea92346f85444c3a8d2f6d5681381830] to archive 2024-12-03T04:45:24,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:45:24,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/0fa1b9b2638e4dc781d241744a13ed00 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/0fa1b9b2638e4dc781d241744a13ed00 2024-12-03T04:45:24,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7d16c770c9204df183962baff52df763 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7d16c770c9204df183962baff52df763 2024-12-03T04:45:24,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/5ae4ea290c9e458c996ea959c6f990e2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/5ae4ea290c9e458c996ea959c6f990e2 2024-12-03T04:45:24,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/abe513bbed4148e199aae94f378cb1da to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/abe513bbed4148e199aae94f378cb1da 2024-12-03T04:45:24,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/b34b2952b5a04b769be74ea300a9727f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/b34b2952b5a04b769be74ea300a9727f 2024-12-03T04:45:24,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7e95852ed9f74f3facb8c245f61302df to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7e95852ed9f74f3facb8c245f61302df 2024-12-03T04:45:24,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/99d6f41cda7a49e49b96f16d357c4bef to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/99d6f41cda7a49e49b96f16d357c4bef 2024-12-03T04:45:24,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/18a67a99d1e949d3a70cd1c8438d5a4f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/18a67a99d1e949d3a70cd1c8438d5a4f 2024-12-03T04:45:24,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/1e3264fa8d7e47aeaa2b9d2fc63c6b1b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/1e3264fa8d7e47aeaa2b9d2fc63c6b1b 2024-12-03T04:45:24,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c44e84b997b0496f9aadef25fbbc2d86 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c44e84b997b0496f9aadef25fbbc2d86 2024-12-03T04:45:24,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/274420c727394b9198b386e51dcd1f70 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/274420c727394b9198b386e51dcd1f70 2024-12-03T04:45:24,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/cc9562829a364e4298eb5c7d1d7ceae4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/cc9562829a364e4298eb5c7d1d7ceae4 2024-12-03T04:45:24,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c615d7e3e73c4574a9c91e11349f4e92 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/c615d7e3e73c4574a9c91e11349f4e92 2024-12-03T04:45:24,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/66995955a24e4ce1a7cb90d7550e0d6c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/66995955a24e4ce1a7cb90d7550e0d6c 2024-12-03T04:45:24,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7a6c1a4ed2c94470bc675b87932359c3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/7a6c1a4ed2c94470bc675b87932359c3 2024-12-03T04:45:24,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/983d8dcc999e4c498284e5b441caf439 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/983d8dcc999e4c498284e5b441caf439 2024-12-03T04:45:24,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/2a49a0a3674741869d5b15403b38391e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/2a49a0a3674741869d5b15403b38391e 2024-12-03T04:45:24,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/1ff868be12684465b88e5a24b7e52b7e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/1ff868be12684465b88e5a24b7e52b7e 2024-12-03T04:45:24,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fce5dd374c6245b09890f85cc3bbbd9c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fce5dd374c6245b09890f85cc3bbbd9c 2024-12-03T04:45:24,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/979f3cd6980a4dfb8a157eb48c0ac9ab to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/979f3cd6980a4dfb8a157eb48c0ac9ab 2024-12-03T04:45:24,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/3dbeea5d5b2e47158fe10215362c7ae9 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/3dbeea5d5b2e47158fe10215362c7ae9 2024-12-03T04:45:24,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fb922055487d4a7fb97cb4227decca51 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/fb922055487d4a7fb97cb4227decca51 2024-12-03T04:45:24,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/da8fa1a55f8847c5958a9de531884af4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/da8fa1a55f8847c5958a9de531884af4 2024-12-03T04:45:24,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/ea92346f85444c3a8d2f6d5681381830 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/ea92346f85444c3a8d2f6d5681381830 2024-12-03T04:45:24,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/9956d2128e9d4c86baac841c26050bca, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/b7983e9f9d4e4269bd1da3bb56f07568, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/cdc4a1a0439e4b60b79420203d5e0425, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/df397e3a34f1423fb1a6bb6d86bc5e8a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/8076e12e3c554ae89f8bde24fcd818a4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ae84e94916c4c6eb191b555d337972e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/eb8d7a56ec304f009199a32c8188a881, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/e048586513ef44a68d9152c54fb7820c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1c5b907b070743229892fd2e17701504, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/61cda50394cb404ab90de76b83f3b9b7, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/0bc6345f6c6f4252926799b97563ff07, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c933cb3d05ce4bb7a4d347ec8e909c49, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/28d04c31551c4ad98c8888deb3a596aa, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/eea72fa3b0a94413994a5a0e18b08718, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/0a3c8376f5764c69932d6428367080f0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ff1103344c143b6b885ca10a2151f0e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/cd43bf0b6cd94b568ec05555f3e77a5e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c38c1421e4db4ff0b015ec6f593f64a3, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1a7b5849f91a4d80947535c0255d5f1c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/10c194a32cae4a049f24fbf1fa011bd9, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/fc089f41899f405a8820313b5a8ab03e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/52122df04a934fee8684dfbe7f72ba0d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/d277e431bc184c48a0ba1b1506de01f7] to archive 2024-12-03T04:45:24,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:45:24,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/9956d2128e9d4c86baac841c26050bca to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/9956d2128e9d4c86baac841c26050bca 2024-12-03T04:45:24,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/b7983e9f9d4e4269bd1da3bb56f07568 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/b7983e9f9d4e4269bd1da3bb56f07568 2024-12-03T04:45:24,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/cdc4a1a0439e4b60b79420203d5e0425 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/cdc4a1a0439e4b60b79420203d5e0425 2024-12-03T04:45:24,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/df397e3a34f1423fb1a6bb6d86bc5e8a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/df397e3a34f1423fb1a6bb6d86bc5e8a 2024-12-03T04:45:24,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/8076e12e3c554ae89f8bde24fcd818a4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/8076e12e3c554ae89f8bde24fcd818a4 2024-12-03T04:45:24,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ae84e94916c4c6eb191b555d337972e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ae84e94916c4c6eb191b555d337972e 2024-12-03T04:45:24,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/eb8d7a56ec304f009199a32c8188a881 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/eb8d7a56ec304f009199a32c8188a881 2024-12-03T04:45:24,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/e048586513ef44a68d9152c54fb7820c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/e048586513ef44a68d9152c54fb7820c 2024-12-03T04:45:24,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1c5b907b070743229892fd2e17701504 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1c5b907b070743229892fd2e17701504 2024-12-03T04:45:24,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/61cda50394cb404ab90de76b83f3b9b7 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/61cda50394cb404ab90de76b83f3b9b7 2024-12-03T04:45:24,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/0bc6345f6c6f4252926799b97563ff07 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/0bc6345f6c6f4252926799b97563ff07 2024-12-03T04:45:24,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c933cb3d05ce4bb7a4d347ec8e909c49 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c933cb3d05ce4bb7a4d347ec8e909c49 2024-12-03T04:45:24,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/28d04c31551c4ad98c8888deb3a596aa to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/28d04c31551c4ad98c8888deb3a596aa 2024-12-03T04:45:24,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/eea72fa3b0a94413994a5a0e18b08718 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/eea72fa3b0a94413994a5a0e18b08718 2024-12-03T04:45:24,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/0a3c8376f5764c69932d6428367080f0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/0a3c8376f5764c69932d6428367080f0 2024-12-03T04:45:24,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ff1103344c143b6b885ca10a2151f0e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/3ff1103344c143b6b885ca10a2151f0e 2024-12-03T04:45:24,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/cd43bf0b6cd94b568ec05555f3e77a5e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/cd43bf0b6cd94b568ec05555f3e77a5e 2024-12-03T04:45:24,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c38c1421e4db4ff0b015ec6f593f64a3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/c38c1421e4db4ff0b015ec6f593f64a3 2024-12-03T04:45:24,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1a7b5849f91a4d80947535c0255d5f1c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/1a7b5849f91a4d80947535c0255d5f1c 2024-12-03T04:45:24,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/10c194a32cae4a049f24fbf1fa011bd9 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/10c194a32cae4a049f24fbf1fa011bd9 2024-12-03T04:45:24,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/fc089f41899f405a8820313b5a8ab03e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/fc089f41899f405a8820313b5a8ab03e 2024-12-03T04:45:24,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/52122df04a934fee8684dfbe7f72ba0d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/52122df04a934fee8684dfbe7f72ba0d 2024-12-03T04:45:24,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/d277e431bc184c48a0ba1b1506de01f7 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/d277e431bc184c48a0ba1b1506de01f7 2024-12-03T04:45:24,933 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/recovered.edits/515.seqid, newMaxSeqId=515, maxSeqId=1 2024-12-03T04:45:24,934 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. 2024-12-03T04:45:24,934 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for a9660218570dfb53504238e06a5637bb: Waiting for close lock at 1733201123613Running coprocessor pre-close hooks at 1733201123613Disabling compacts and flushes for region at 1733201123613Disabling writes for close at 1733201123613Obtaining lock to block concurrent updates at 1733201123613Preparing flush snapshotting stores in a9660218570dfb53504238e06a5637bb at 1733201123613Finished memstore snapshotting TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733201123613Flushing stores of TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb. at 1733201123613Flushing a9660218570dfb53504238e06a5637bb/A: creating writer at 1733201123613Flushing a9660218570dfb53504238e06a5637bb/A: appending metadata at 1733201123616 (+3 ms)Flushing a9660218570dfb53504238e06a5637bb/A: closing flushed file at 1733201123616Flushing a9660218570dfb53504238e06a5637bb/B: creating writer at 1733201124023 (+407 ms)Flushing a9660218570dfb53504238e06a5637bb/B: appending metadata at 1733201124025 (+2 ms)Flushing a9660218570dfb53504238e06a5637bb/B: closing flushed file at 1733201124025Flushing a9660218570dfb53504238e06a5637bb/C: creating writer at 1733201124434 (+409 ms)Flushing a9660218570dfb53504238e06a5637bb/C: appending metadata at 1733201124437 (+3 ms)Flushing a9660218570dfb53504238e06a5637bb/C: closing flushed file at 1733201124437Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@354aaa07: reopening flushed file at 1733201124851 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d95cc6: reopening flushed file at 1733201124857 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22c734d6: reopening flushed file at 1733201124862 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a9660218570dfb53504238e06a5637bb in 1253ms, sequenceid=512, compaction requested=true at 1733201124867 (+5 ms)Writing region close event to WAL at 1733201124931 (+64 ms)Running coprocessor post-close hooks at 1733201124934 (+3 ms)Closed at 1733201124934 2024-12-03T04:45:24,936 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:24,936 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=a9660218570dfb53504238e06a5637bb, regionState=CLOSED 2024-12-03T04:45:24,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:45:24,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-03T04:45:24,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure a9660218570dfb53504238e06a5637bb, server=32beccd76b35,43193,1733200986730 in 1.4770 sec 2024-12-03T04:45:24,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-12-03T04:45:24,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9660218570dfb53504238e06a5637bb, UNASSIGN in 1.4810 sec 2024-12-03T04:45:24,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-03T04:45:24,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4840 sec 2024-12-03T04:45:24,943 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201124943"}]},"ts":"1733201124943"} 2024-12-03T04:45:24,944 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T04:45:24,944 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T04:45:24,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4920 sec 2024-12-03T04:45:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T04:45:25,602 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:25,603 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T04:45:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:25,604 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T04:45:25,605 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:25,607 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:25,609 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/recovered.edits] 2024-12-03T04:45:25,611 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/0067edee795943dbb1561bc5003fefbc to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/0067edee795943dbb1561bc5003fefbc 2024-12-03T04:45:25,612 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/448335e6770a40b9a8cf265dbc4d4536 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/448335e6770a40b9a8cf265dbc4d4536 2024-12-03T04:45:25,613 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e310a185a62f444cb4d33994c38f88f4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/A/e310a185a62f444cb4d33994c38f88f4 2024-12-03T04:45:25,615 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/82a936ce61a841fdb7a49a26eb3208d6 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/82a936ce61a841fdb7a49a26eb3208d6 2024-12-03T04:45:25,616 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/905a655b673f4ef3a392c2862998dcde to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/905a655b673f4ef3a392c2862998dcde 2024-12-03T04:45:25,617 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/bb428a11befd465dac180ad69f4a60f4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/B/bb428a11befd465dac180ad69f4a60f4 2024-12-03T04:45:25,618 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/011ae5f1db29480c8c1a0634f228d2d0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/011ae5f1db29480c8c1a0634f228d2d0 2024-12-03T04:45:25,619 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/ca65e2f38fac40e9a6b96822fdf752c5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/ca65e2f38fac40e9a6b96822fdf752c5 2024-12-03T04:45:25,619 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/ce263b1cf5024f0ba1f4cac3243886be to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/C/ce263b1cf5024f0ba1f4cac3243886be 2024-12-03T04:45:25,621 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/recovered.edits/515.seqid to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb/recovered.edits/515.seqid 2024-12-03T04:45:25,622 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/a9660218570dfb53504238e06a5637bb 2024-12-03T04:45:25,622 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T04:45:25,623 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:25,625 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T04:45:25,627 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T04:45:25,628 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:25,628 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T04:45:25,628 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733201125628"}]},"ts":"9223372036854775807"} 2024-12-03T04:45:25,629 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T04:45:25,629 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a9660218570dfb53504238e06a5637bb, NAME => 'TestAcidGuarantees,,1733201099242.a9660218570dfb53504238e06a5637bb.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T04:45:25,629 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T04:45:25,629 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733201125629"}]},"ts":"9223372036854775807"} 2024-12-03T04:45:25,631 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T04:45:25,631 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:25,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-12-03T04:45:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T04:45:25,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T04:45:25,712 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:25,721 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 250), OpenFileDescriptor=459 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=739 (was 838), ProcessCount=11 (was 11), AvailableMemoryMB=3361 (was 3528) 2024-12-03T04:45:25,730 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=739, ProcessCount=11, AvailableMemoryMB=3360 2024-12-03T04:45:25,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T04:45:25,732 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:45:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:25,734 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T04:45:25,734 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:25,734 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-12-03T04:45:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-03T04:45:25,735 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T04:45:25,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742375_1551 (size=1061) 2024-12-03T04:45:25,745 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676 2024-12-03T04:45:25,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742376_1552 (size=53) 2024-12-03T04:45:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-03T04:45:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-03T04:45:26,155 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:45:26,155 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5585179eb550a7f04eee9f770b5bea8c, disabling compactions & flushes 2024-12-03T04:45:26,155 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:26,155 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:26,155 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. after waiting 0 ms 2024-12-03T04:45:26,155 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:26,155 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:26,155 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5585179eb550a7f04eee9f770b5bea8c: Waiting for close lock at 1733201126155Disabling compacts and flushes for region at 1733201126155Disabling writes for close at 1733201126155Writing region close event to WAL at 1733201126155Closed at 1733201126155 2024-12-03T04:45:26,156 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T04:45:26,156 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733201126156"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733201126156"}]},"ts":"1733201126156"} 2024-12-03T04:45:26,158 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T04:45:26,158 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T04:45:26,158 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201126158"}]},"ts":"1733201126158"} 2024-12-03T04:45:26,159 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T04:45:26,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, ASSIGN}] 2024-12-03T04:45:26,160 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, ASSIGN 2024-12-03T04:45:26,161 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, ASSIGN; state=OFFLINE, location=32beccd76b35,43193,1733200986730; forceNewPlan=false, retain=false 2024-12-03T04:45:26,311 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=5585179eb550a7f04eee9f770b5bea8c, regionState=OPENING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:45:26,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, ASSIGN because future has completed 2024-12-03T04:45:26,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:45:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-03T04:45:26,467 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:26,467 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:45:26,467 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,467 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:45:26,468 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,468 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,469 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,470 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:45:26,470 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5585179eb550a7f04eee9f770b5bea8c columnFamilyName A 2024-12-03T04:45:26,470 DEBUG [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:26,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T04:45:26,470 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(327): Store=5585179eb550a7f04eee9f770b5bea8c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:45:26,471 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,471 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:45:26,472 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5585179eb550a7f04eee9f770b5bea8c columnFamilyName B 2024-12-03T04:45:26,472 DEBUG [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:26,472 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(327): Store=5585179eb550a7f04eee9f770b5bea8c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:45:26,472 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,473 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:45:26,473 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5585179eb550a7f04eee9f770b5bea8c columnFamilyName C 2024-12-03T04:45:26,473 DEBUG [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:26,473 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(327): Store=5585179eb550a7f04eee9f770b5bea8c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:45:26,473 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:26,473 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,474 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,474 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,475 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,475 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,475 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:45:26,476 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,477 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T04:45:26,477 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened 5585179eb550a7f04eee9f770b5bea8c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61026527, jitterRate=-0.09063388407230377}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:45:26,478 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:26,478 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for 5585179eb550a7f04eee9f770b5bea8c: Running coprocessor pre-open hook at 1733201126468Writing region info on filesystem at 1733201126468Initializing all the Stores at 1733201126468Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201126468Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201126469 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201126469Cleaning up temporary data from old regions at 1733201126475 (+6 ms)Running coprocessor post-open hooks at 1733201126478 (+3 ms)Region opened successfully at 1733201126478 2024-12-03T04:45:26,479 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., pid=136, masterSystemTime=1733201126465 2024-12-03T04:45:26,480 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:26,480 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:26,480 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=5585179eb550a7f04eee9f770b5bea8c, regionState=OPEN, openSeqNum=2, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:45:26,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:45:26,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-03T04:45:26,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 in 169 msec 2024-12-03T04:45:26,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-03T04:45:26,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, ASSIGN in 324 msec 2024-12-03T04:45:26,485 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T04:45:26,485 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201126485"}]},"ts":"1733201126485"} 2024-12-03T04:45:26,486 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T04:45:26,486 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T04:45:26,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 754 msec 2024-12-03T04:45:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-03T04:45:26,872 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:26,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66320673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:26,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:26,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:26,874 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:26,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4149e27e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:26,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:26,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:26,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:26,876 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@268edc2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:26,877 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:26,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:26,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:26,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:26,879 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T04:45:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 32beccd76b35,38397,1733200986071 2024-12-03T04:45:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a3227ed 2024-12-03T04:45:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T04:45:26,881 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T04:45:26,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T04:45:26,882 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T04:45:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:26,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742377_1553 (size=1097) 2024-12-03T04:45:27,293 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T04:45:27,293 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T04:45:27,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T04:45:27,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, REOPEN/MOVE}] 2024-12-03T04:45:27,297 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, REOPEN/MOVE 2024-12-03T04:45:27,298 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=5585179eb550a7f04eee9f770b5bea8c, regionState=CLOSING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, REOPEN/MOVE because future has completed 2024-12-03T04:45:27,299 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T04:45:27,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:45:27,450 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,450 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T04:45:27,450 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 5585179eb550a7f04eee9f770b5bea8c, disabling compactions & flushes 2024-12-03T04:45:27,450 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:27,451 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:27,451 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. after waiting 0 ms 2024-12-03T04:45:27,451 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:27,453 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T04:45:27,454 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:27,454 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 5585179eb550a7f04eee9f770b5bea8c: Waiting for close lock at 1733201127450Running coprocessor pre-close hooks at 1733201127450Disabling compacts and flushes for region at 1733201127450Disabling writes for close at 1733201127451 (+1 ms)Writing region close event to WAL at 1733201127451Running coprocessor post-close hooks at 1733201127454 (+3 ms)Closed at 1733201127454 2024-12-03T04:45:27,454 WARN [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: 5585179eb550a7f04eee9f770b5bea8c to self. 2024-12-03T04:45:27,456 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,456 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=5585179eb550a7f04eee9f770b5bea8c, regionState=CLOSED 2024-12-03T04:45:27,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:45:27,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-03T04:45:27,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 in 159 msec 2024-12-03T04:45:27,459 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, REOPEN/MOVE; state=CLOSED, location=32beccd76b35,43193,1733200986730; forceNewPlan=false, retain=true 2024-12-03T04:45:27,610 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=5585179eb550a7f04eee9f770b5bea8c, regionState=OPENING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, REOPEN/MOVE because future has completed 2024-12-03T04:45:27,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:45:27,766 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:27,766 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} 2024-12-03T04:45:27,766 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,766 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T04:45:27,766 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,766 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,767 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,768 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:45:27,768 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5585179eb550a7f04eee9f770b5bea8c columnFamilyName A 2024-12-03T04:45:27,769 DEBUG [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:27,769 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(327): Store=5585179eb550a7f04eee9f770b5bea8c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:45:27,770 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,770 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:45:27,770 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5585179eb550a7f04eee9f770b5bea8c columnFamilyName B 2024-12-03T04:45:27,770 DEBUG [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:27,771 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(327): Store=5585179eb550a7f04eee9f770b5bea8c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:45:27,771 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,771 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T04:45:27,771 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5585179eb550a7f04eee9f770b5bea8c columnFamilyName C 2024-12-03T04:45:27,771 DEBUG [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:27,771 INFO [StoreOpener-5585179eb550a7f04eee9f770b5bea8c-1 {}] regionserver.HStore(327): Store=5585179eb550a7f04eee9f770b5bea8c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T04:45:27,772 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:27,772 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,772 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,773 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,773 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,773 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,774 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T04:45:27,775 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,775 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened 5585179eb550a7f04eee9f770b5bea8c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70401231, jitterRate=0.049060091376304626}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T04:45:27,775 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,776 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for 5585179eb550a7f04eee9f770b5bea8c: Running coprocessor pre-open hook at 1733201127766Writing region info on filesystem at 1733201127766Initializing all the Stores at 1733201127767 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201127767Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201127767Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733201127767Cleaning up temporary data from old regions at 1733201127773 (+6 ms)Running coprocessor post-open hooks at 1733201127775 (+2 ms)Region opened successfully at 1733201127776 (+1 ms) 2024-12-03T04:45:27,776 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., pid=141, masterSystemTime=1733201127763 2024-12-03T04:45:27,778 DEBUG [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:27,778 INFO [RS_OPEN_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:27,778 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=5585179eb550a7f04eee9f770b5bea8c, regionState=OPEN, openSeqNum=5, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:45:27,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-12-03T04:45:27,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 in 169 msec 2024-12-03T04:45:27,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-03T04:45:27,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, REOPEN/MOVE in 485 msec 2024-12-03T04:45:27,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-03T04:45:27,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-12-03T04:45:27,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 902 msec 2024-12-03T04:45:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T04:45:27,787 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:27,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:27,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:27,788 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58acd824, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:27,790 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:27,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37cfcd0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,791 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:27,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:27,791 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:27,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e7cf76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:27,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:27,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:27,792 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:27,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c805cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:27,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:27,794 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8b0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:27,795 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a7194cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:27,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:27,797 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3016775d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:27,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:27,798 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:27,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@281591b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:27,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:27,799 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47e055d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:27,800 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a911933, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:27,802 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:27,802 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f187105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:27,803 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e6a69bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:27,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:27,805 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5582c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:27,806 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f1a4dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:27,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:27,807 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c259b79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:27,809 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f54497b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:27,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:27,810 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@639b5210, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:27,811 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c2e3e77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32beccd76b35,38397,-1 for getting cluster id 2024-12-03T04:45:27,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T04:45:27,813 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08870cf8-d835-4c97-9170-0203a4d6da45' 2024-12-03T04:45:27,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T04:45:27,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08870cf8-d835-4c97-9170-0203a4d6da45" 2024-12-03T04:45:27,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56367f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32beccd76b35,38397,-1] 2024-12-03T04:45:27,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T04:45:27,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:27,814 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T04:45:27,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d08725b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T04:45:27,815 DEBUG [Thread-2466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:27,815 DEBUG [Thread-2472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:27,815 DEBUG [Thread-2470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:27,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:27,816 DEBUG [Thread-2464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-12-03T04:45:27,816 DEBUG [Thread-2455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:27,816 DEBUG [Thread-2457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:27,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:27,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T04:45:27,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:27,818 DEBUG [Thread-2464 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:27,818 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:27,818 DEBUG [Thread-2464 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:27,818 DEBUG [Thread-2461 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:27,818 DEBUG [Thread-2453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:27,819 DEBUG [Thread-2459 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:27,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:27,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:45:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:27,819 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:27,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:27,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:27,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:45:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:27,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:27,822 DEBUG [Thread-2468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T04:45:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:45:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:45:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:45:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32beccd76b35,43193,1733200986730, seqNum=-1] 2024-12-03T04:45:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T04:45:27,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:45:27,824 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T04:45:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:45:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:45:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:45:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5] 2024-12-03T04:45:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:45:27,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:27,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:27,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:27,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:27,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:27,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:27,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038460796d60b4476db2ecd57ecb0cf0a6_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201127827/Put/seqid=0 2024-12-03T04:45:27,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742378_1554 (size=17034) 2024-12-03T04:45:27,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201127846, max timestamp=1733201127853Num uniques 859; 2024-12-03T04:45:27,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201127846, max timestamp=1733201127853Num uniques 859; 2024-12-03T04:45:27,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:27,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201127846, max timestamp=1733201127853Num uniques 859; 2024-12-03T04:45:27,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:27,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:27,857 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:27,860 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038460796d60b4476db2ecd57ecb0cf0a6_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038460796d60b4476db2ecd57ecb0cf0a6_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:27,861 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/2855e8634d704be19842f3448fb76950, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:27,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/2855e8634d704be19842f3448fb76950 is 175, key is test_row_0/A:col10/1733201127827/Put/seqid=0 2024-12-03T04:45:27,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742379_1555 (size=48139) 2024-12-03T04:45:27,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201187871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:27,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201187873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201187874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:27,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201187875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:27,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201187877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T04:45:27,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T04:45:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:27,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:27,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201187981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201187981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:27,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201187982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201187982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:27,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201187992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:27,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:27,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:28,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T04:45:28,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:28,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T04:45:28,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201188192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:28,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201188192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:28,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201188192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:28,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201188192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201188201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:28,272 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/2855e8634d704be19842f3448fb76950 2024-12-03T04:45:28,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T04:45:28,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:28,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/7743445d60f1448fabf8272830b447eb is 50, key is test_row_0/B:col10/1733201127827/Put/seqid=0 2024-12-03T04:45:28,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742380_1556 (size=12001) 2024-12-03T04:45:28,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T04:45:28,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:28,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T04:45:28,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201188502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:28,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201188502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:28,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201188502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:28,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201188502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:28,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201188511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:28,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:28,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T04:45:28,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:28,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/7743445d60f1448fabf8272830b447eb 2024-12-03T04:45:28,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/77199354f2344eb2adb943faa54e0f09 is 50, key is test_row_0/C:col10/1733201127827/Put/seqid=0 2024-12-03T04:45:28,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T04:45:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742381_1557 (size=12001) 2024-12-03T04:45:28,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T04:45:28,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:28,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T04:45:29,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201189011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:29,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201189011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:29,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201189011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:29,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201189013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:29,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201189025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:29,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T04:45:29,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:29,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:29,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:29,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:29,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:29,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/77199354f2344eb2adb943faa54e0f09 2024-12-03T04:45:29,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/2855e8634d704be19842f3448fb76950 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/2855e8634d704be19842f3448fb76950 2024-12-03T04:45:29,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/2855e8634d704be19842f3448fb76950, entries=250, sequenceid=15, filesize=47.0 K 2024-12-03T04:45:29,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/7743445d60f1448fabf8272830b447eb as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7743445d60f1448fabf8272830b447eb 2024-12-03T04:45:29,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T04:45:29,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:29,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:29,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:29,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7743445d60f1448fabf8272830b447eb, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T04:45:29,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:29,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:29,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/77199354f2344eb2adb943faa54e0f09 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/77199354f2344eb2adb943faa54e0f09 2024-12-03T04:45:29,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/77199354f2344eb2adb943faa54e0f09, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T04:45:29,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5585179eb550a7f04eee9f770b5bea8c in 1407ms, sequenceid=15, compaction requested=false 2024-12-03T04:45:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T04:45:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:29,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T04:45:29,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:29,381 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T04:45:29,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:29,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:29,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:29,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:29,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:29,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:29,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cb92f80dcf85409e98ecc108b2cfd037_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201127864/Put/seqid=0 2024-12-03T04:45:29,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742382_1558 (size=12154) 2024-12-03T04:45:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:29,797 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cb92f80dcf85409e98ecc108b2cfd037_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cb92f80dcf85409e98ecc108b2cfd037_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:29,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/dbd474ee2b454ee08bc445f679a1d412, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:29,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/dbd474ee2b454ee08bc445f679a1d412 is 175, key is test_row_0/A:col10/1733201127864/Put/seqid=0 2024-12-03T04:45:29,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742383_1559 (size=30955) 2024-12-03T04:45:29,803 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/dbd474ee2b454ee08bc445f679a1d412 2024-12-03T04:45:29,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/fc3b98ca33bc48d1997d4df27bff61ac is 50, key is test_row_0/B:col10/1733201127864/Put/seqid=0 2024-12-03T04:45:29,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742384_1560 (size=12001) 2024-12-03T04:45:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T04:45:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:30,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:30,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201190033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201190034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201190036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201190036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201190041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,046 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:30,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201190152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201190152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201190152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201190152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,215 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/fc3b98ca33bc48d1997d4df27bff61ac 2024-12-03T04:45:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/52999967c95b4ee782097913d44ab179 is 50, key is test_row_0/C:col10/1733201127864/Put/seqid=0 2024-12-03T04:45:30,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742385_1561 (size=12001) 2024-12-03T04:45:30,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201190361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201190361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201190361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201190361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,462 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T04:45:30,628 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/52999967c95b4ee782097913d44ab179 2024-12-03T04:45:30,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/dbd474ee2b454ee08bc445f679a1d412 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/dbd474ee2b454ee08bc445f679a1d412 2024-12-03T04:45:30,634 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/dbd474ee2b454ee08bc445f679a1d412, entries=150, sequenceid=51, filesize=30.2 K 2024-12-03T04:45:30,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/fc3b98ca33bc48d1997d4df27bff61ac as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/fc3b98ca33bc48d1997d4df27bff61ac 2024-12-03T04:45:30,638 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/fc3b98ca33bc48d1997d4df27bff61ac, entries=150, sequenceid=51, filesize=11.7 K 2024-12-03T04:45:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/52999967c95b4ee782097913d44ab179 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/52999967c95b4ee782097913d44ab179 2024-12-03T04:45:30,644 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/52999967c95b4ee782097913d44ab179, entries=150, sequenceid=51, filesize=11.7 K 2024-12-03T04:45:30,645 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 5585179eb550a7f04eee9f770b5bea8c in 1264ms, sequenceid=51, compaction requested=false 2024-12-03T04:45:30,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:30,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:30,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-12-03T04:45:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-12-03T04:45:30,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-03T04:45:30,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8280 sec 2024-12-03T04:45:30,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.8330 sec 2024-12-03T04:45:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:30,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T04:45:30,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:30,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:30,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:30,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:30,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:30,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:30,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030b02087b07044f12a4c8f520c01fe0ea_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201130032/Put/seqid=0 2024-12-03T04:45:30,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742386_1562 (size=12154) 2024-12-03T04:45:30,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201190695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201190699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201190700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201190700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201190801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201190811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201190811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201190812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201191011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201191021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201191022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201191022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,085 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:31,088 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030b02087b07044f12a4c8f520c01fe0ea_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030b02087b07044f12a4c8f520c01fe0ea_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:31,095 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0b4078d42d0b4c04a94fe20ee5679f2d, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:31,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0b4078d42d0b4c04a94fe20ee5679f2d is 175, key is test_row_0/A:col10/1733201130032/Put/seqid=0 2024-12-03T04:45:31,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742387_1563 (size=30955) 2024-12-03T04:45:31,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201191321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201191332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201191332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201191332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,500 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0b4078d42d0b4c04a94fe20ee5679f2d 2024-12-03T04:45:31,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/272dc9c39b8d48b999f1328b9564d845 is 50, key is test_row_0/B:col10/1733201130032/Put/seqid=0 2024-12-03T04:45:31,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742388_1564 (size=12001) 2024-12-03T04:45:31,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201191831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201191842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201191842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201191851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:31,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/272dc9c39b8d48b999f1328b9564d845 2024-12-03T04:45:31,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/5d1fb28e140f4035a9add5de1dcdc6b5 is 50, key is test_row_0/C:col10/1733201130032/Put/seqid=0 2024-12-03T04:45:31,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742389_1565 (size=12001) 2024-12-03T04:45:31,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/5d1fb28e140f4035a9add5de1dcdc6b5 2024-12-03T04:45:31,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0b4078d42d0b4c04a94fe20ee5679f2d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0b4078d42d0b4c04a94fe20ee5679f2d 2024-12-03T04:45:31,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0b4078d42d0b4c04a94fe20ee5679f2d, entries=150, sequenceid=64, filesize=30.2 K 2024-12-03T04:45:31,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/272dc9c39b8d48b999f1328b9564d845 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/272dc9c39b8d48b999f1328b9564d845 2024-12-03T04:45:31,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/272dc9c39b8d48b999f1328b9564d845, entries=150, sequenceid=64, filesize=11.7 K 2024-12-03T04:45:31,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/5d1fb28e140f4035a9add5de1dcdc6b5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/5d1fb28e140f4035a9add5de1dcdc6b5 2024-12-03T04:45:31,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/5d1fb28e140f4035a9add5de1dcdc6b5, entries=150, sequenceid=64, filesize=11.7 K 2024-12-03T04:45:31,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5585179eb550a7f04eee9f770b5bea8c in 1281ms, sequenceid=64, compaction requested=true 2024-12-03T04:45:31,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:31,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:31,956 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:31,956 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:31,956 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:31,956 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/A is initiating minor compaction (all files) 2024-12-03T04:45:31,956 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/A in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:31,957 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/2855e8634d704be19842f3448fb76950, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/dbd474ee2b454ee08bc445f679a1d412, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0b4078d42d0b4c04a94fe20ee5679f2d] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=107.5 K 2024-12-03T04:45:31,957 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:31,957 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/2855e8634d704be19842f3448fb76950, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/dbd474ee2b454ee08bc445f679a1d412, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0b4078d42d0b4c04a94fe20ee5679f2d] 2024-12-03T04:45:31,957 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:31,957 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/B is initiating minor compaction (all files) 2024-12-03T04:45:31,957 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/B in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:31,957 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7743445d60f1448fabf8272830b447eb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/fc3b98ca33bc48d1997d4df27bff61ac, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/272dc9c39b8d48b999f1328b9564d845] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=35.2 K 2024-12-03T04:45:31,957 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2855e8634d704be19842f3448fb76950, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733201127823 2024-12-03T04:45:31,957 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 7743445d60f1448fabf8272830b447eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733201127825 2024-12-03T04:45:31,958 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting fc3b98ca33bc48d1997d4df27bff61ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733201127864 2024-12-03T04:45:31,958 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbd474ee2b454ee08bc445f679a1d412, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733201127864 2024-12-03T04:45:31,958 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 272dc9c39b8d48b999f1328b9564d845, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733201130025 2024-12-03T04:45:31,958 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b4078d42d0b4c04a94fe20ee5679f2d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733201130025 2024-12-03T04:45:31,964 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:31,965 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203e5c8a29adfd34682967823c7cfd60801_5585179eb550a7f04eee9f770b5bea8c store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:31,966 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#B#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:31,966 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/3755ffae8bcb45c0b48054c865d749b5 is 50, key is test_row_0/B:col10/1733201130032/Put/seqid=0 2024-12-03T04:45:31,967 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203e5c8a29adfd34682967823c7cfd60801_5585179eb550a7f04eee9f770b5bea8c, store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:31,967 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e5c8a29adfd34682967823c7cfd60801_5585179eb550a7f04eee9f770b5bea8c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:31,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742390_1566 (size=12104) 2024-12-03T04:45:31,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742391_1567 (size=4469) 2024-12-03T04:45:31,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T04:45:31,982 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:31,983 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/3755ffae8bcb45c0b48054c865d749b5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/3755ffae8bcb45c0b48054c865d749b5 2024-12-03T04:45:31,983 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#A#compaction#484 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:31,983 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:31,984 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/d4fe66ed4c3747ff98387575db536731 is 175, key is test_row_0/A:col10/1733201130032/Put/seqid=0 2024-12-03T04:45:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-03T04:45:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T04:45:31,985 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:31,986 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:31,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:31,988 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/B of 5585179eb550a7f04eee9f770b5bea8c into 3755ffae8bcb45c0b48054c865d749b5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:31,988 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:31,988 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/B, priority=13, startTime=1733201131956; duration=0sec 2024-12-03T04:45:31,988 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:31,988 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:B 2024-12-03T04:45:31,988 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T04:45:31,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742392_1568 (size=31058) 2024-12-03T04:45:31,989 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T04:45:31,989 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/C is initiating minor compaction (all files) 2024-12-03T04:45:31,989 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/C in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:31,989 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/77199354f2344eb2adb943faa54e0f09, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/52999967c95b4ee782097913d44ab179, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/5d1fb28e140f4035a9add5de1dcdc6b5] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=35.2 K 2024-12-03T04:45:31,990 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 77199354f2344eb2adb943faa54e0f09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733201127825 2024-12-03T04:45:31,990 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 52999967c95b4ee782097913d44ab179, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733201127864 2024-12-03T04:45:31,990 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1fb28e140f4035a9add5de1dcdc6b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733201130025 2024-12-03T04:45:32,000 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#C#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:32,000 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/b343e4ec3a9747c5b827fb012f60eeaa is 50, key is test_row_0/C:col10/1733201130032/Put/seqid=0 2024-12-03T04:45:32,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742393_1569 (size=12104) 2024-12-03T04:45:32,012 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/b343e4ec3a9747c5b827fb012f60eeaa as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/b343e4ec3a9747c5b827fb012f60eeaa 2024-12-03T04:45:32,015 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/C of 5585179eb550a7f04eee9f770b5bea8c into b343e4ec3a9747c5b827fb012f60eeaa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:32,015 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:32,016 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/C, priority=13, startTime=1733201131956; duration=0sec 2024-12-03T04:45:32,016 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:32,016 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:C 2024-12-03T04:45:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:32,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:45:32,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:32,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:32,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:32,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:32,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:32,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:32,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037d1481d915234d709b7e4b06c7a6c4ef_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201132063/Put/seqid=0 2024-12-03T04:45:32,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742394_1570 (size=12154) 2024-12-03T04:45:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T04:45:32,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201192104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:32,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T04:45:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201192212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:32,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T04:45:32,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:32,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T04:45:32,393 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/d4fe66ed4c3747ff98387575db536731 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d4fe66ed4c3747ff98387575db536731 2024-12-03T04:45:32,397 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/A of 5585179eb550a7f04eee9f770b5bea8c into d4fe66ed4c3747ff98387575db536731(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:32,397 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:32,397 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/A, priority=13, startTime=1733201131955; duration=0sec 2024-12-03T04:45:32,397 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:32,397 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:A 2024-12-03T04:45:32,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201192421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:32,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:32,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T04:45:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,474 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:32,477 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037d1481d915234d709b7e4b06c7a6c4ef_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037d1481d915234d709b7e4b06c7a6c4ef_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:32,478 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/e0251da0987447b4a71e4090738f5830, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:32,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/e0251da0987447b4a71e4090738f5830 is 175, key is test_row_0/A:col10/1733201132063/Put/seqid=0 2024-12-03T04:45:32,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742395_1571 (size=30955) 2024-12-03T04:45:32,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T04:45:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T04:45:32,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201192732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:32,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T04:45:32,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201192842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:32,845 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:32,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:32,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201192851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:32,855 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:32,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201192862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:32,866 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:32,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:32,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201192872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:32,874 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:32,890 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/e0251da0987447b4a71e4090738f5830 2024-12-03T04:45:32,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T04:45:32,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:32,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:32,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:32,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/1530d89c8ce24d34aca979783ba9f649 is 50, key is test_row_0/B:col10/1733201132063/Put/seqid=0 2024-12-03T04:45:32,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742396_1572 (size=12001) 2024-12-03T04:45:33,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T04:45:33,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:33,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:33,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:33,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:33,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T04:45:33,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T04:45:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:33,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:33,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:33,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201193245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:33,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/1530d89c8ce24d34aca979783ba9f649 2024-12-03T04:45:33,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/2769ea71f0d64aefbafe33702951ffa2 is 50, key is test_row_0/C:col10/1733201132063/Put/seqid=0 2024-12-03T04:45:33,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T04:45:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:33,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:33,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:33,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:33,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:33,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742397_1573 (size=12001) 2024-12-03T04:45:33,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/2769ea71f0d64aefbafe33702951ffa2 2024-12-03T04:45:33,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/e0251da0987447b4a71e4090738f5830 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/e0251da0987447b4a71e4090738f5830 2024-12-03T04:45:33,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/e0251da0987447b4a71e4090738f5830, entries=150, sequenceid=86, filesize=30.2 K 2024-12-03T04:45:33,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/1530d89c8ce24d34aca979783ba9f649 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/1530d89c8ce24d34aca979783ba9f649 2024-12-03T04:45:33,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/1530d89c8ce24d34aca979783ba9f649, entries=150, sequenceid=86, filesize=11.7 K 2024-12-03T04:45:33,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/2769ea71f0d64aefbafe33702951ffa2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/2769ea71f0d64aefbafe33702951ffa2 2024-12-03T04:45:33,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/2769ea71f0d64aefbafe33702951ffa2, entries=150, sequenceid=86, filesize=11.7 K 2024-12-03T04:45:33,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5585179eb550a7f04eee9f770b5bea8c in 1330ms, sequenceid=86, compaction requested=false 2024-12-03T04:45:33,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:33,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T04:45:33,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:33,517 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:45:33,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:33,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:33,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:33,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:33,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:33,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f6f81f3fdbaa45ac847a149fd1982033_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201132102/Put/seqid=0 2024-12-03T04:45:33,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742398_1574 (size=12154) 2024-12-03T04:45:33,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:33,539 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f6f81f3fdbaa45ac847a149fd1982033_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f6f81f3fdbaa45ac847a149fd1982033_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/4a70bacba16b443d997c0be57ce460da, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:33,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/4a70bacba16b443d997c0be57ce460da is 175, key is test_row_0/A:col10/1733201132102/Put/seqid=0 2024-12-03T04:45:33,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742399_1575 (size=30955) 2024-12-03T04:45:33,945 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/4a70bacba16b443d997c0be57ce460da 2024-12-03T04:45:33,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/a7f833bb1cc147e9af6017ac1c3fca8e is 50, key is test_row_0/B:col10/1733201132102/Put/seqid=0 2024-12-03T04:45:33,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742400_1576 (size=12001) 2024-12-03T04:45:33,975 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/a7f833bb1cc147e9af6017ac1c3fca8e 2024-12-03T04:45:33,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/3e8f06b733954cb089206e95c6f79f02 is 50, key is test_row_0/C:col10/1733201132102/Put/seqid=0 2024-12-03T04:45:33,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742401_1577 (size=12001) 2024-12-03T04:45:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T04:45:34,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:34,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:34,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:34,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201194347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:34,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:34,396 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/3e8f06b733954cb089206e95c6f79f02 2024-12-03T04:45:34,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/4a70bacba16b443d997c0be57ce460da as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/4a70bacba16b443d997c0be57ce460da 2024-12-03T04:45:34,404 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/4a70bacba16b443d997c0be57ce460da, entries=150, sequenceid=99, filesize=30.2 K 2024-12-03T04:45:34,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/a7f833bb1cc147e9af6017ac1c3fca8e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/a7f833bb1cc147e9af6017ac1c3fca8e 2024-12-03T04:45:34,410 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/a7f833bb1cc147e9af6017ac1c3fca8e, entries=150, sequenceid=99, filesize=11.7 K 2024-12-03T04:45:34,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/3e8f06b733954cb089206e95c6f79f02 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/3e8f06b733954cb089206e95c6f79f02 2024-12-03T04:45:34,414 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/3e8f06b733954cb089206e95c6f79f02, entries=150, sequenceid=99, filesize=11.7 K 2024-12-03T04:45:34,414 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5585179eb550a7f04eee9f770b5bea8c in 897ms, sequenceid=99, compaction requested=true 2024-12-03T04:45:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-03T04:45:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-03T04:45:34,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-03T04:45:34,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4300 sec 2024-12-03T04:45:34,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.4350 sec 2024-12-03T04:45:34,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201134318, max timestamp=1733201134464Num uniques 859; 2024-12-03T04:45:34,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:34,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201134318, max timestamp=1733201134464Num uniques 859; 2024-12-03T04:45:34,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:34,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-03T04:45:34,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:34,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:34,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:34,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:34,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:34,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:34,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120318ecac6baed1438eba4aaedf4a8d4015_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201134318/Put/seqid=0 2024-12-03T04:45:34,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742402_1578 (size=14594) 2024-12-03T04:45:34,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201134545, max timestamp=1733201134564Num uniques 859; 2024-12-03T04:45:34,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201134545, max timestamp=1733201134564Num uniques 859; 2024-12-03T04:45:34,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:34,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:34,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201134545, max timestamp=1733201134564Num uniques 859; 2024-12-03T04:45:34,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:34,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201194674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:34,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:34,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201194782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:34,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201194861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:34,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201194861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:34,863 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:34,863 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:34,884 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201194883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:34,886 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:34,888 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120318ecac6baed1438eba4aaedf4a8d4015_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120318ecac6baed1438eba4aaedf4a8d4015_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:34,889 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/c39778dcc3f84f5ea1f37999cae1a6bd, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:34,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/c39778dcc3f84f5ea1f37999cae1a6bd is 175, key is test_row_0/A:col10/1733201134318/Put/seqid=0 2024-12-03T04:45:34,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742403_1579 (size=39549) 2024-12-03T04:45:34,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201194902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:34,905 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:34,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:34,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201194992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:35,164 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T04:45:35,303 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/c39778dcc3f84f5ea1f37999cae1a6bd 2024-12-03T04:45:35,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201195302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:35,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:35,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:35,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:35,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/d9845a1b7ff3452d8cf5de4f7a46efe4 is 50, key is test_row_0/B:col10/1733201134318/Put/seqid=0 2024-12-03T04:45:35,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742404_1580 (size=12001) 2024-12-03T04:45:35,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/d9845a1b7ff3452d8cf5de4f7a46efe4 2024-12-03T04:45:35,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/1d805bace17a4f24a76c6f656ae4f731 is 50, key is test_row_0/C:col10/1733201134318/Put/seqid=0 2024-12-03T04:45:35,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742405_1581 (size=12001) 2024-12-03T04:45:35,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/1d805bace17a4f24a76c6f656ae4f731 2024-12-03T04:45:35,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/c39778dcc3f84f5ea1f37999cae1a6bd as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/c39778dcc3f84f5ea1f37999cae1a6bd 2024-12-03T04:45:35,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/c39778dcc3f84f5ea1f37999cae1a6bd, entries=200, sequenceid=120, filesize=38.6 K 2024-12-03T04:45:35,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/d9845a1b7ff3452d8cf5de4f7a46efe4 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/d9845a1b7ff3452d8cf5de4f7a46efe4 2024-12-03T04:45:35,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/d9845a1b7ff3452d8cf5de4f7a46efe4, entries=150, sequenceid=120, filesize=11.7 K 2024-12-03T04:45:35,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/1d805bace17a4f24a76c6f656ae4f731 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/1d805bace17a4f24a76c6f656ae4f731 2024-12-03T04:45:35,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/1d805bace17a4f24a76c6f656ae4f731, entries=150, sequenceid=120, filesize=11.7 K 2024-12-03T04:45:35,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 5585179eb550a7f04eee9f770b5bea8c in 1270ms, sequenceid=120, compaction requested=true 2024-12-03T04:45:35,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:35,739 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:35,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:35,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:35,740 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:35,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:35,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:35,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:35,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:35,741 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:35,741 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/A is initiating minor compaction (all files) 2024-12-03T04:45:35,741 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/A in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:35,741 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d4fe66ed4c3747ff98387575db536731, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/e0251da0987447b4a71e4090738f5830, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/4a70bacba16b443d997c0be57ce460da, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/c39778dcc3f84f5ea1f37999cae1a6bd] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=129.4 K 2024-12-03T04:45:35,741 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:35,741 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d4fe66ed4c3747ff98387575db536731, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/e0251da0987447b4a71e4090738f5830, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/4a70bacba16b443d997c0be57ce460da, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/c39778dcc3f84f5ea1f37999cae1a6bd] 2024-12-03T04:45:35,742 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4fe66ed4c3747ff98387575db536731, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733201130025 2024-12-03T04:45:35,742 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0251da0987447b4a71e4090738f5830, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733201130688 2024-12-03T04:45:35,742 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:35,742 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a70bacba16b443d997c0be57ce460da, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733201132083 2024-12-03T04:45:35,743 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/B is initiating minor compaction (all files) 2024-12-03T04:45:35,743 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/B in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:35,743 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/3755ffae8bcb45c0b48054c865d749b5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/1530d89c8ce24d34aca979783ba9f649, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/a7f833bb1cc147e9af6017ac1c3fca8e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/d9845a1b7ff3452d8cf5de4f7a46efe4] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=47.0 K 2024-12-03T04:45:35,743 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting c39778dcc3f84f5ea1f37999cae1a6bd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733201134318 2024-12-03T04:45:35,743 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3755ffae8bcb45c0b48054c865d749b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733201130025 2024-12-03T04:45:35,743 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 1530d89c8ce24d34aca979783ba9f649, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733201130688 2024-12-03T04:45:35,744 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a7f833bb1cc147e9af6017ac1c3fca8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733201132083 2024-12-03T04:45:35,744 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting d9845a1b7ff3452d8cf5de4f7a46efe4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733201134318 2024-12-03T04:45:35,752 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:35,754 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203593631e8c8ab4b1caf1e87c0b673c701_5585179eb550a7f04eee9f770b5bea8c store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:35,755 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#B#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:35,755 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/41857dfd2d6c4fffbd439fb8c58f0722 is 50, key is test_row_0/B:col10/1733201134318/Put/seqid=0 2024-12-03T04:45:35,757 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203593631e8c8ab4b1caf1e87c0b673c701_5585179eb550a7f04eee9f770b5bea8c, store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:35,757 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203593631e8c8ab4b1caf1e87c0b673c701_5585179eb550a7f04eee9f770b5bea8c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:35,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742406_1582 (size=12241) 2024-12-03T04:45:35,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742407_1583 (size=4469) 2024-12-03T04:45:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:35,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T04:45:35,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:35,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:35,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:35,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:35,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:35,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:35,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c8f4ae9cdb4a4da5a43c972bb337f64d_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201134631/Put/seqid=0 2024-12-03T04:45:35,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742408_1584 (size=14794) 2024-12-03T04:45:35,827 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:35,832 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c8f4ae9cdb4a4da5a43c972bb337f64d_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c8f4ae9cdb4a4da5a43c972bb337f64d_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:35,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/71830851771f46ada9f78ebaa68d5c5a, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:35,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/71830851771f46ada9f78ebaa68d5c5a is 175, key is test_row_0/A:col10/1733201134631/Put/seqid=0 2024-12-03T04:45:35,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742409_1585 (size=39749) 2024-12-03T04:45:35,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201195875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:35,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:35,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:35,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:35,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201195982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T04:45:36,152 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:36,152 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-03T04:45:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T04:45:36,154 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:36,155 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:36,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:36,164 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/41857dfd2d6c4fffbd439fb8c58f0722 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/41857dfd2d6c4fffbd439fb8c58f0722 2024-12-03T04:45:36,167 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/B of 5585179eb550a7f04eee9f770b5bea8c into 41857dfd2d6c4fffbd439fb8c58f0722(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:36,167 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:36,167 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/B, priority=12, startTime=1733201135740; duration=0sec 2024-12-03T04:45:36,167 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:36,167 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:B 2024-12-03T04:45:36,168 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:36,169 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:36,169 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/C is initiating minor compaction (all files) 2024-12-03T04:45:36,169 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/C in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:36,169 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/b343e4ec3a9747c5b827fb012f60eeaa, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/2769ea71f0d64aefbafe33702951ffa2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/3e8f06b733954cb089206e95c6f79f02, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/1d805bace17a4f24a76c6f656ae4f731] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=47.0 K 2024-12-03T04:45:36,169 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting b343e4ec3a9747c5b827fb012f60eeaa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733201130025 2024-12-03T04:45:36,170 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 2769ea71f0d64aefbafe33702951ffa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733201130688 2024-12-03T04:45:36,170 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e8f06b733954cb089206e95c6f79f02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733201132083 2024-12-03T04:45:36,170 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d805bace17a4f24a76c6f656ae4f731, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733201134318 2024-12-03T04:45:36,171 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#A#compaction#496 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:36,171 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/55c235c4c285488e8f2f45f144c99da0 is 175, key is test_row_0/A:col10/1733201134318/Put/seqid=0 2024-12-03T04:45:36,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742410_1586 (size=31195) 2024-12-03T04:45:36,178 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#C#compaction#499 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:36,178 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/9c95c58ba69748ca823c955857ba6b9c is 50, key is test_row_0/C:col10/1733201134318/Put/seqid=0 2024-12-03T04:45:36,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742411_1587 (size=12241) 2024-12-03T04:45:36,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201196191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:36,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:36,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:36,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:36,239 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/71830851771f46ada9f78ebaa68d5c5a 2024-12-03T04:45:36,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/06db24845e1a4ecaa42a33d6b11f3fd5 is 50, key is test_row_0/B:col10/1733201134631/Put/seqid=0 2024-12-03T04:45:36,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742412_1588 (size=12151) 2024-12-03T04:45:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T04:45:36,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T04:45:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:36,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T04:45:36,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:36,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:36,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:36,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T04:45:36,470 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T04:45:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T04:45:36,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201196502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:36,579 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/55c235c4c285488e8f2f45f144c99da0 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/55c235c4c285488e8f2f45f144c99da0 2024-12-03T04:45:36,583 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/A of 5585179eb550a7f04eee9f770b5bea8c into 55c235c4c285488e8f2f45f144c99da0(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:36,583 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:36,583 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/A, priority=12, startTime=1733201135739; duration=0sec 2024-12-03T04:45:36,583 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:36,583 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:A 2024-12-03T04:45:36,585 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/9c95c58ba69748ca823c955857ba6b9c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/9c95c58ba69748ca823c955857ba6b9c 2024-12-03T04:45:36,589 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/C of 5585179eb550a7f04eee9f770b5bea8c into 9c95c58ba69748ca823c955857ba6b9c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:36,589 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:36,589 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/C, priority=12, startTime=1733201135740; duration=0sec 2024-12-03T04:45:36,589 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:36,589 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:C 2024-12-03T04:45:36,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T04:45:36,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:36,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/06db24845e1a4ecaa42a33d6b11f3fd5 2024-12-03T04:45:36,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/025c750a9545445cb78c496d6976c5a2 is 50, key is test_row_0/C:col10/1733201134631/Put/seqid=0 2024-12-03T04:45:36,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742413_1589 (size=12151) 2024-12-03T04:45:36,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T04:45:36,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:36,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:36,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:36,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T04:45:36,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T04:45:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:36,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:37,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:37,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201197011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:37,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/025c750a9545445cb78c496d6976c5a2 2024-12-03T04:45:37,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T04:45:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:37,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:37,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/71830851771f46ada9f78ebaa68d5c5a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/71830851771f46ada9f78ebaa68d5c5a 2024-12-03T04:45:37,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/71830851771f46ada9f78ebaa68d5c5a, entries=200, sequenceid=157, filesize=38.8 K 2024-12-03T04:45:37,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/06db24845e1a4ecaa42a33d6b11f3fd5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/06db24845e1a4ecaa42a33d6b11f3fd5 2024-12-03T04:45:37,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/06db24845e1a4ecaa42a33d6b11f3fd5, entries=150, sequenceid=157, filesize=11.9 K 2024-12-03T04:45:37,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/025c750a9545445cb78c496d6976c5a2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/025c750a9545445cb78c496d6976c5a2 2024-12-03T04:45:37,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/025c750a9545445cb78c496d6976c5a2, entries=150, sequenceid=157, filesize=11.9 K 2024-12-03T04:45:37,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 5585179eb550a7f04eee9f770b5bea8c in 1270ms, sequenceid=157, compaction requested=false 2024-12-03T04:45:37,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:37,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T04:45:37,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:37,224 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:45:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:37,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035c957b8e3b52490db08ef7463854a16c_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201135873/Put/seqid=0 2024-12-03T04:45:37,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742414_1590 (size=12304) 2024-12-03T04:45:37,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,280 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035c957b8e3b52490db08ef7463854a16c_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c957b8e3b52490db08ef7463854a16c_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:37,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/d973ae082a49479786b41c2a0ac6d41c, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/d973ae082a49479786b41c2a0ac6d41c is 175, key is test_row_0/A:col10/1733201135873/Put/seqid=0 2024-12-03T04:45:37,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742415_1591 (size=31105) 2024-12-03T04:45:37,286 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/d973ae082a49479786b41c2a0ac6d41c 2024-12-03T04:45:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T04:45:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/92becd190a9e40d2a6619f6b1b3ab016 is 50, key is test_row_0/B:col10/1733201135873/Put/seqid=0 2024-12-03T04:45:37,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742416_1592 (size=12151) 2024-12-03T04:45:37,312 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/92becd190a9e40d2a6619f6b1b3ab016 2024-12-03T04:45:37,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/f2142837f53c4542bc6e6b5c42c8334b is 50, key is test_row_0/C:col10/1733201135873/Put/seqid=0 2024-12-03T04:45:37,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742417_1593 (size=12151) 2024-12-03T04:45:37,336 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/f2142837f53c4542bc6e6b5c42c8334b 2024-12-03T04:45:37,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/d973ae082a49479786b41c2a0ac6d41c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d973ae082a49479786b41c2a0ac6d41c 2024-12-03T04:45:37,356 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d973ae082a49479786b41c2a0ac6d41c, entries=150, sequenceid=170, filesize=30.4 K 2024-12-03T04:45:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/92becd190a9e40d2a6619f6b1b3ab016 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/92becd190a9e40d2a6619f6b1b3ab016 2024-12-03T04:45:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,361 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/92becd190a9e40d2a6619f6b1b3ab016, entries=150, sequenceid=170, filesize=11.9 K 2024-12-03T04:45:37,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/f2142837f53c4542bc6e6b5c42c8334b as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/f2142837f53c4542bc6e6b5c42c8334b 2024-12-03T04:45:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,367 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/f2142837f53c4542bc6e6b5c42c8334b, entries=150, sequenceid=170, filesize=11.9 K 2024-12-03T04:45:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,368 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 5585179eb550a7f04eee9f770b5bea8c in 144ms, sequenceid=170, compaction requested=true 2024-12-03T04:45:37,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:37,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:37,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-03T04:45:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-03T04:45:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-03T04:45:37,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2140 sec 2024-12-03T04:45:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.2200 sec 2024-12-03T04:45:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:38,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:45:38,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:38,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:38,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:38,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:38,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:38,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:38,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120360ccb16f545d45a1a0ec975d7293edd3_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201138104/Put/seqid=0 2024-12-03T04:45:38,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742418_1594 (size=12304) 2024-12-03T04:45:38,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201138184, max timestamp=1733201138203Num uniques 859; 2024-12-03T04:45:38,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:38,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201138184, max timestamp=1733201138203Num uniques 859; 2024-12-03T04:45:38,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:38,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201138184, max timestamp=1733201138203Num uniques 859; 2024-12-03T04:45:38,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T04:45:38,312 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:38,313 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-03T04:45:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T04:45:38,314 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:38,315 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:38,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:38,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201198311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T04:45:38,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201198422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:38,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T04:45:38,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:38,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:38,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:38,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,521 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:38,524 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120360ccb16f545d45a1a0ec975d7293edd3_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120360ccb16f545d45a1a0ec975d7293edd3_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:38,525 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0a69e4822156465bad7ecf0f03e811fc, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:38,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0a69e4822156465bad7ecf0f03e811fc is 175, key is test_row_0/A:col10/1733201138104/Put/seqid=0 2024-12-03T04:45:38,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742419_1595 (size=31105) 2024-12-03T04:45:38,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T04:45:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:38,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T04:45:38,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201198632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:38,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:38,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T04:45:38,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:38,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:38,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:38,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44414 deadline: 1733201198891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:38,896 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:38,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44460 deadline: 1733201198901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:38,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44442 deadline: 1733201198901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:38,906 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_0' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:38,907 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:38,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T04:45:38,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:38,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:38,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:38,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:38,930 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0a69e4822156465bad7ecf0f03e811fc 2024-12-03T04:45:38,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/6c850a1dcb344fbdb826fd3765ff91d0 is 50, key is test_row_0/B:col10/1733201138104/Put/seqid=0 2024-12-03T04:45:38,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44452 deadline: 1733201198932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:38,936 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32beccd76b35,43193,1733200986730 for 'test_row_2' in 5585179eb550a7f04eee9f770b5bea8c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-03T04:45:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:38,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742420_1596 (size=12151) 2024-12-03T04:45:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T04:45:38,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201198941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:39,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T04:45:39,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:39,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:39,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:39,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:39,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:39,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T04:45:39,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:39,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:39,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:39,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:39,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:39,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/6c850a1dcb344fbdb826fd3765ff91d0 2024-12-03T04:45:39,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/efc8b9771cf7470091f37e910dc356dd is 50, key is test_row_0/C:col10/1733201138104/Put/seqid=0 2024-12-03T04:45:39,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742421_1597 (size=12151) 2024-12-03T04:45:39,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/efc8b9771cf7470091f37e910dc356dd 2024-12-03T04:45:39,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0a69e4822156465bad7ecf0f03e811fc as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0a69e4822156465bad7ecf0f03e811fc 2024-12-03T04:45:39,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0a69e4822156465bad7ecf0f03e811fc, entries=150, sequenceid=180, filesize=30.4 K 2024-12-03T04:45:39,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/6c850a1dcb344fbdb826fd3765ff91d0 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/6c850a1dcb344fbdb826fd3765ff91d0 2024-12-03T04:45:39,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/6c850a1dcb344fbdb826fd3765ff91d0, entries=150, sequenceid=180, filesize=11.9 K 2024-12-03T04:45:39,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/efc8b9771cf7470091f37e910dc356dd as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/efc8b9771cf7470091f37e910dc356dd 2024-12-03T04:45:39,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/efc8b9771cf7470091f37e910dc356dd, entries=150, sequenceid=180, filesize=11.9 K 2024-12-03T04:45:39,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5585179eb550a7f04eee9f770b5bea8c in 1256ms, sequenceid=180, compaction requested=true 2024-12-03T04:45:39,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:39,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:39,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:39,366 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:39,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:39,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:39,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:39,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:39,366 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:39,367 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:39,367 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:39,367 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/A is initiating minor compaction (all files) 2024-12-03T04:45:39,367 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/B is initiating minor compaction (all files) 2024-12-03T04:45:39,367 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/A in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:39,367 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/B in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:39,367 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/55c235c4c285488e8f2f45f144c99da0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/71830851771f46ada9f78ebaa68d5c5a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d973ae082a49479786b41c2a0ac6d41c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0a69e4822156465bad7ecf0f03e811fc] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=130.0 K 2024-12-03T04:45:39,367 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/41857dfd2d6c4fffbd439fb8c58f0722, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/06db24845e1a4ecaa42a33d6b11f3fd5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/92becd190a9e40d2a6619f6b1b3ab016, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/6c850a1dcb344fbdb826fd3765ff91d0] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=47.6 K 2024-12-03T04:45:39,367 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:39,367 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/55c235c4c285488e8f2f45f144c99da0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/71830851771f46ada9f78ebaa68d5c5a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d973ae082a49479786b41c2a0ac6d41c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0a69e4822156465bad7ecf0f03e811fc] 2024-12-03T04:45:39,367 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 41857dfd2d6c4fffbd439fb8c58f0722, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733201134318 2024-12-03T04:45:39,367 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55c235c4c285488e8f2f45f144c99da0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733201134318 2024-12-03T04:45:39,367 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71830851771f46ada9f78ebaa68d5c5a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733201134631 2024-12-03T04:45:39,367 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 06db24845e1a4ecaa42a33d6b11f3fd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733201134631 2024-12-03T04:45:39,368 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting d973ae082a49479786b41c2a0ac6d41c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733201135841 2024-12-03T04:45:39,368 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 92becd190a9e40d2a6619f6b1b3ab016, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733201135841 2024-12-03T04:45:39,368 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a69e4822156465bad7ecf0f03e811fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201138100 2024-12-03T04:45:39,368 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c850a1dcb344fbdb826fd3765ff91d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201138100 2024-12-03T04:45:39,373 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:39,378 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412039d97116f30504fe485568c21f988e7b8_5585179eb550a7f04eee9f770b5bea8c store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:39,379 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412039d97116f30504fe485568c21f988e7b8_5585179eb550a7f04eee9f770b5bea8c, store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:39,379 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039d97116f30504fe485568c21f988e7b8_5585179eb550a7f04eee9f770b5bea8c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:39,380 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#B#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:39,380 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/82e23d1b599f4c209d51ad7e8287fc2d is 50, key is test_row_0/B:col10/1733201138104/Put/seqid=0 2024-12-03T04:45:39,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742422_1598 (size=4469) 2024-12-03T04:45:39,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T04:45:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:39,385 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T04:45:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:39,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742423_1599 (size=12527) 2024-12-03T04:45:39,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203057c27f74ec844359b082e5c9335285b_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201138295/Put/seqid=0 2024-12-03T04:45:39,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742424_1600 (size=12304) 2024-12-03T04:45:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T04:45:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:39,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:39,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201199493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:39,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201199602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:39,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:39,784 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#A#compaction#508 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:39,785 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/de2be6e8cee84ee8a9f71f32c48d4345 is 175, key is test_row_0/A:col10/1733201138104/Put/seqid=0 2024-12-03T04:45:39,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742425_1601 (size=31481) 2024-12-03T04:45:39,795 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/82e23d1b599f4c209d51ad7e8287fc2d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/82e23d1b599f4c209d51ad7e8287fc2d 2024-12-03T04:45:39,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:39,799 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/B of 5585179eb550a7f04eee9f770b5bea8c into 82e23d1b599f4c209d51ad7e8287fc2d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:39,799 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:39,799 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/B, priority=12, startTime=1733201139366; duration=0sec 2024-12-03T04:45:39,799 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:39,799 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:B 2024-12-03T04:45:39,800 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:39,801 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:39,801 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/C is initiating minor compaction (all files) 2024-12-03T04:45:39,801 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/C in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:39,801 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/9c95c58ba69748ca823c955857ba6b9c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/025c750a9545445cb78c496d6976c5a2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/f2142837f53c4542bc6e6b5c42c8334b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/efc8b9771cf7470091f37e910dc356dd] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=47.6 K 2024-12-03T04:45:39,802 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c95c58ba69748ca823c955857ba6b9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733201134318 2024-12-03T04:45:39,802 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 025c750a9545445cb78c496d6976c5a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733201134631 2024-12-03T04:45:39,802 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting f2142837f53c4542bc6e6b5c42c8334b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733201135841 2024-12-03T04:45:39,802 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203057c27f74ec844359b082e5c9335285b_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203057c27f74ec844359b082e5c9335285b_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:39,802 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting efc8b9771cf7470091f37e910dc356dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201138100 2024-12-03T04:45:39,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/22491b9e703d4b2db988ae5172c72199, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:39,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/22491b9e703d4b2db988ae5172c72199 is 175, key is test_row_0/A:col10/1733201138295/Put/seqid=0 2024-12-03T04:45:39,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742426_1602 (size=31105) 2024-12-03T04:45:39,809 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/22491b9e703d4b2db988ae5172c72199 2024-12-03T04:45:39,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201199811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:39,814 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#C#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:39,814 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/348043d7090d49f28ab940aa982648ab is 50, key is test_row_0/C:col10/1733201138104/Put/seqid=0 2024-12-03T04:45:39,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/b2f71e3c2d18408ea459942c201d0b95 is 50, key is test_row_0/B:col10/1733201138295/Put/seqid=0 2024-12-03T04:45:39,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742427_1603 (size=12527) 2024-12-03T04:45:39,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:39,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742428_1604 (size=12151) 2024-12-03T04:45:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:40,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201200132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:40,193 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/de2be6e8cee84ee8a9f71f32c48d4345 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/de2be6e8cee84ee8a9f71f32c48d4345 2024-12-03T04:45:40,198 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/A of 5585179eb550a7f04eee9f770b5bea8c into de2be6e8cee84ee8a9f71f32c48d4345(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:40,198 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:40,198 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/A, priority=12, startTime=1733201139366; duration=0sec 2024-12-03T04:45:40,198 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:40,199 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:A 2024-12-03T04:45:40,221 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/b2f71e3c2d18408ea459942c201d0b95 2024-12-03T04:45:40,221 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/348043d7090d49f28ab940aa982648ab as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/348043d7090d49f28ab940aa982648ab 2024-12-03T04:45:40,226 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/C of 5585179eb550a7f04eee9f770b5bea8c into 348043d7090d49f28ab940aa982648ab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:40,226 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:40,227 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/C, priority=12, startTime=1733201139366; duration=0sec 2024-12-03T04:45:40,227 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:40,227 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:C 2024-12-03T04:45:40,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/cc2ff7c66dc04159b3c66d012bbdb0ee is 50, key is test_row_0/C:col10/1733201138295/Put/seqid=0 2024-12-03T04:45:40,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742429_1605 (size=12151) 2024-12-03T04:45:40,236 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/cc2ff7c66dc04159b3c66d012bbdb0ee 2024-12-03T04:45:40,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/22491b9e703d4b2db988ae5172c72199 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/22491b9e703d4b2db988ae5172c72199 2024-12-03T04:45:40,249 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/22491b9e703d4b2db988ae5172c72199, entries=150, sequenceid=217, filesize=30.4 K 2024-12-03T04:45:40,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/b2f71e3c2d18408ea459942c201d0b95 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/b2f71e3c2d18408ea459942c201d0b95 2024-12-03T04:45:40,254 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/b2f71e3c2d18408ea459942c201d0b95, entries=150, sequenceid=217, filesize=11.9 K 2024-12-03T04:45:40,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/cc2ff7c66dc04159b3c66d012bbdb0ee as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cc2ff7c66dc04159b3c66d012bbdb0ee 2024-12-03T04:45:40,258 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cc2ff7c66dc04159b3c66d012bbdb0ee, entries=150, sequenceid=217, filesize=11.9 K 2024-12-03T04:45:40,259 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 5585179eb550a7f04eee9f770b5bea8c in 874ms, sequenceid=217, compaction requested=false 2024-12-03T04:45:40,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:40,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:40,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-03T04:45:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-03T04:45:40,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-03T04:45:40,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9450 sec 2024-12-03T04:45:40,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.9500 sec 2024-12-03T04:45:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T04:45:40,462 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:40,466 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-03T04:45:40,468 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T04:45:40,469 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:40,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T04:45:40,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T04:45:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:40,622 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T04:45:40,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:40,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:40,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:40,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:40,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:40,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:40,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b0e0ba5811fb408c97d668eed30c9148_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201139478/Put/seqid=0 2024-12-03T04:45:40,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742430_1606 (size=12304) 2024-12-03T04:45:40,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:40,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:40,647 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b0e0ba5811fb408c97d668eed30c9148_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b0e0ba5811fb408c97d668eed30c9148_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:40,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/96a167988b8945298e974c9b973b2034, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:40,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/96a167988b8945298e974c9b973b2034 is 175, key is test_row_0/A:col10/1733201139478/Put/seqid=0 2024-12-03T04:45:40,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742431_1607 (size=31105) 2024-12-03T04:45:40,665 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/96a167988b8945298e974c9b973b2034 2024-12-03T04:45:40,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/07b52e1b247a4dd2bb18868f2763ca3c is 50, key is test_row_0/B:col10/1733201139478/Put/seqid=0 2024-12-03T04:45:40,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742432_1608 (size=12151) 2024-12-03T04:45:40,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201140701, max timestamp=1733201140739Num uniques 859; 2024-12-03T04:45:40,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:40,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201140701, max timestamp=1733201140739Num uniques 859; 2024-12-03T04:45:40,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:40,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733201140701, max timestamp=1733201140739Num uniques 859; 2024-12-03T04:45:40,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32beccd76b35:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T04:45:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T04:45:40,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201200841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:40,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:40,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:40,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:40,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:40,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201200952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:41,091 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/07b52e1b247a4dd2bb18868f2763ca3c 2024-12-03T04:45:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T04:45:41,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/03f4aac4165747359e6f0b9dbcebd965 is 50, key is test_row_0/C:col10/1733201139478/Put/seqid=0 2024-12-03T04:45:41,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742433_1609 (size=12151) 2024-12-03T04:45:41,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201201161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:41,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201201472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:41,503 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/03f4aac4165747359e6f0b9dbcebd965 2024-12-03T04:45:41,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/96a167988b8945298e974c9b973b2034 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/96a167988b8945298e974c9b973b2034 2024-12-03T04:45:41,509 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/96a167988b8945298e974c9b973b2034, entries=150, sequenceid=230, filesize=30.4 K 2024-12-03T04:45:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/07b52e1b247a4dd2bb18868f2763ca3c as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/07b52e1b247a4dd2bb18868f2763ca3c 2024-12-03T04:45:41,513 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/07b52e1b247a4dd2bb18868f2763ca3c, entries=150, sequenceid=230, filesize=11.9 K 2024-12-03T04:45:41,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/03f4aac4165747359e6f0b9dbcebd965 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/03f4aac4165747359e6f0b9dbcebd965 2024-12-03T04:45:41,516 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/03f4aac4165747359e6f0b9dbcebd965, entries=150, sequenceid=230, filesize=11.9 K 2024-12-03T04:45:41,517 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5585179eb550a7f04eee9f770b5bea8c in 895ms, sequenceid=230, compaction requested=true 2024-12-03T04:45:41,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:41,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:41,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-03T04:45:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-03T04:45:41,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-03T04:45:41,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0490 sec 2024-12-03T04:45:41,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.0540 sec 2024-12-03T04:45:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T04:45:41,602 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:41,603 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-03T04:45:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T04:45:41,605 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:41,605 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:41,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T04:45:41,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T04:45:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:41,758 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T04:45:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:41,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T04:45:41,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a20fb5ddb82c42d6ab39184e0d07ba69_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201140818/Put/seqid=0 2024-12-03T04:45:41,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742434_1610 (size=12454) 2024-12-03T04:45:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T04:45:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:42,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201202075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,173 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a20fb5ddb82c42d6ab39184e0d07ba69_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a20fb5ddb82c42d6ab39184e0d07ba69_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:42,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/729ddfaead64477abf85e460ecac5d13, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:42,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/729ddfaead64477abf85e460ecac5d13 is 175, key is test_row_0/A:col10/1733201140818/Put/seqid=0 2024-12-03T04:45:42,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201202193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:42,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742435_1611 (size=31255) 2024-12-03T04:45:42,209 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/729ddfaead64477abf85e460ecac5d13 2024-12-03T04:45:42,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/86ceeccfab9f4151ae68836b6491476d is 50, key is test_row_0/B:col10/1733201140818/Put/seqid=0 2024-12-03T04:45:42,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742436_1612 (size=12301) 2024-12-03T04:45:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T04:45:42,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201202402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:42,622 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/86ceeccfab9f4151ae68836b6491476d 2024-12-03T04:45:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/902d0e67d81840f6821f199ca9a01a80 is 50, key is test_row_0/C:col10/1733201140818/Put/seqid=0 2024-12-03T04:45:42,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742437_1613 (size=12301) 2024-12-03T04:45:42,634 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/902d0e67d81840f6821f199ca9a01a80 2024-12-03T04:45:42,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/729ddfaead64477abf85e460ecac5d13 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/729ddfaead64477abf85e460ecac5d13 2024-12-03T04:45:42,641 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/729ddfaead64477abf85e460ecac5d13, entries=150, sequenceid=267, filesize=30.5 K 2024-12-03T04:45:42,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/86ceeccfab9f4151ae68836b6491476d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/86ceeccfab9f4151ae68836b6491476d 2024-12-03T04:45:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,645 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/86ceeccfab9f4151ae68836b6491476d, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T04:45:42,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/902d0e67d81840f6821f199ca9a01a80 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/902d0e67d81840f6821f199ca9a01a80 2024-12-03T04:45:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,649 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/902d0e67d81840f6821f199ca9a01a80, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T04:45:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,651 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 5585179eb550a7f04eee9f770b5bea8c in 892ms, sequenceid=267, compaction requested=true 2024-12-03T04:45:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:42,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-03T04:45:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-03T04:45:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-03T04:45:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0470 sec 2024-12-03T04:45:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0540 sec 2024-12-03T04:45:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:42,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T04:45:42,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:42,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:42,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:42,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:42,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:42,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203caa35d073ee14265877153548ce0ed10_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201142065/Put/seqid=0 2024-12-03T04:45:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742438_1614 (size=9914) 2024-12-03T04:45:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203caa35d073ee14265877153548ce0ed10_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203caa35d073ee14265877153548ce0ed10_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,740 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/b1c4caf98e8b4cce89cfb2a0b49ad92f, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/b1c4caf98e8b4cce89cfb2a0b49ad92f is 175, key is test_row_0/A:col10/1733201142065/Put/seqid=0 2024-12-03T04:45:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T04:45:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,742 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,742 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-03T04:45:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T04:45:42,744 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,745 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:42,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742439_1615 (size=22561) 2024-12-03T04:45:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,749 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/b1c4caf98e8b4cce89cfb2a0b49ad92f 2024-12-03T04:45:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/c9810086290043b99ba7158b77464a8d is 50, key is test_row_0/B:col10/1733201142065/Put/seqid=0 2024-12-03T04:45:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742440_1616 (size=9857) 2024-12-03T04:45:42,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/c9810086290043b99ba7158b77464a8d 2024-12-03T04:45:42,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/08be618f9a5449df9f2a9d2799fb417d is 50, key is test_row_0/C:col10/1733201142065/Put/seqid=0 2024-12-03T04:45:42,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742441_1617 (size=9857) 2024-12-03T04:45:42,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/08be618f9a5449df9f2a9d2799fb417d 2024-12-03T04:45:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/b1c4caf98e8b4cce89cfb2a0b49ad92f as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/b1c4caf98e8b4cce89cfb2a0b49ad92f 2024-12-03T04:45:42,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/b1c4caf98e8b4cce89cfb2a0b49ad92f, entries=100, sequenceid=278, filesize=22.0 K 2024-12-03T04:45:42,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/c9810086290043b99ba7158b77464a8d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/c9810086290043b99ba7158b77464a8d 2024-12-03T04:45:42,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/c9810086290043b99ba7158b77464a8d, entries=100, sequenceid=278, filesize=9.6 K 2024-12-03T04:45:42,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/08be618f9a5449df9f2a9d2799fb417d as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/08be618f9a5449df9f2a9d2799fb417d 2024-12-03T04:45:42,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/08be618f9a5449df9f2a9d2799fb417d, entries=100, sequenceid=278, filesize=9.6 K 2024-12-03T04:45:42,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=100.63 KB/103050 for 5585179eb550a7f04eee9f770b5bea8c in 118ms, sequenceid=278, compaction requested=true 2024-12-03T04:45:42,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:42,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:42,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:42,834 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T04:45:42,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:42,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:42,834 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T04:45:42,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:42,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:42,835 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 147507 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T04:45:42,835 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58987 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T04:45:42,835 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/A is initiating minor compaction (all files) 2024-12-03T04:45:42,835 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/B is initiating minor compaction (all files) 2024-12-03T04:45:42,835 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/B in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:42,835 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/A in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:42,835 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/82e23d1b599f4c209d51ad7e8287fc2d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/b2f71e3c2d18408ea459942c201d0b95, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/07b52e1b247a4dd2bb18868f2763ca3c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/86ceeccfab9f4151ae68836b6491476d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/c9810086290043b99ba7158b77464a8d] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=57.6 K 2024-12-03T04:45:42,836 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/de2be6e8cee84ee8a9f71f32c48d4345, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/22491b9e703d4b2db988ae5172c72199, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/96a167988b8945298e974c9b973b2034, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/729ddfaead64477abf85e460ecac5d13, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/b1c4caf98e8b4cce89cfb2a0b49ad92f] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=144.0 K 2024-12-03T04:45:42,836 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:42,836 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/de2be6e8cee84ee8a9f71f32c48d4345, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/22491b9e703d4b2db988ae5172c72199, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/96a167988b8945298e974c9b973b2034, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/729ddfaead64477abf85e460ecac5d13, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/b1c4caf98e8b4cce89cfb2a0b49ad92f] 2024-12-03T04:45:42,836 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 82e23d1b599f4c209d51ad7e8287fc2d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201138100 2024-12-03T04:45:42,836 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting de2be6e8cee84ee8a9f71f32c48d4345, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201138100 2024-12-03T04:45:42,836 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f71e3c2d18408ea459942c201d0b95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733201138278 2024-12-03T04:45:42,836 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 07b52e1b247a4dd2bb18868f2763ca3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733201139478 2024-12-03T04:45:42,836 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22491b9e703d4b2db988ae5172c72199, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733201138278 2024-12-03T04:45:42,837 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 86ceeccfab9f4151ae68836b6491476d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733201140818 2024-12-03T04:45:42,837 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96a167988b8945298e974c9b973b2034, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733201139478 2024-12-03T04:45:42,838 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting c9810086290043b99ba7158b77464a8d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733201142065 2024-12-03T04:45:42,838 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 729ddfaead64477abf85e460ecac5d13, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733201140818 2024-12-03T04:45:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:42,839 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1c4caf98e8b4cce89cfb2a0b49ad92f, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733201142065 2024-12-03T04:45:42,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T04:45:42,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:42,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:42,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:42,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:42,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:42,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:42,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037f46f47c1d4a47a68f2f3e94a98a0230_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201142837/Put/seqid=0 2024-12-03T04:45:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T04:45:42,862 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:42,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742442_1618 (size=14994) 2024-12-03T04:45:42,864 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#B#compaction#524 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:42,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:42,865 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/a70f524773fa4e4bafbc80435d9aacec is 50, key is test_row_0/B:col10/1733201142065/Put/seqid=0 2024-12-03T04:45:42,866 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412030c784e77d33f47e19c69f5d922b33e11_5585179eb550a7f04eee9f770b5bea8c store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:42,868 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412030c784e77d33f47e19c69f5d922b33e11_5585179eb550a7f04eee9f770b5bea8c, store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:42,868 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030c784e77d33f47e19c69f5d922b33e11_5585179eb550a7f04eee9f770b5bea8c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:42,872 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037f46f47c1d4a47a68f2f3e94a98a0230_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037f46f47c1d4a47a68f2f3e94a98a0230_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:42,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/7ec145b221af4961b9159457e5049c42, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:42,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/7ec145b221af4961b9159457e5049c42 is 175, key is test_row_0/A:col10/1733201142837/Put/seqid=0 2024-12-03T04:45:42,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742443_1619 (size=12847) 2024-12-03T04:45:42,889 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/a70f524773fa4e4bafbc80435d9aacec as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/a70f524773fa4e4bafbc80435d9aacec 2024-12-03T04:45:42,896 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/B of 5585179eb550a7f04eee9f770b5bea8c into a70f524773fa4e4bafbc80435d9aacec(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:42,896 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:42,896 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/B, priority=11, startTime=1733201142834; duration=0sec 2024-12-03T04:45:42,896 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:42,896 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:B 2024-12-03T04:45:42,896 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T04:45:42,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T04:45:42,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:42,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:42,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:42,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:42,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:42,898 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58987 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T04:45:42,898 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/C is initiating minor compaction (all files) 2024-12-03T04:45:42,898 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/C in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:42,898 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/348043d7090d49f28ab940aa982648ab, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cc2ff7c66dc04159b3c66d012bbdb0ee, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/03f4aac4165747359e6f0b9dbcebd965, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/902d0e67d81840f6821f199ca9a01a80, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/08be618f9a5449df9f2a9d2799fb417d] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=57.6 K 2024-12-03T04:45:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:42,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742444_1620 (size=4469) 2024-12-03T04:45:42,900 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 348043d7090d49f28ab940aa982648ab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733201138100 2024-12-03T04:45:42,900 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting cc2ff7c66dc04159b3c66d012bbdb0ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733201138278 2024-12-03T04:45:42,901 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 03f4aac4165747359e6f0b9dbcebd965, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733201139478 2024-12-03T04:45:42,901 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 902d0e67d81840f6821f199ca9a01a80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733201140818 2024-12-03T04:45:42,902 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 08be618f9a5449df9f2a9d2799fb417d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733201142065 2024-12-03T04:45:42,902 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#A#compaction#525 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:42,903 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/5434d60aedf0463eb83e9c60bf59f880 is 175, key is test_row_0/A:col10/1733201142065/Put/seqid=0 2024-12-03T04:45:42,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742445_1621 (size=39949) 2024-12-03T04:45:42,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/7ec145b221af4961b9159457e5049c42 2024-12-03T04:45:42,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742446_1622 (size=31908) 2024-12-03T04:45:42,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201202907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:42,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/7e293da106a545139ed4d5c142d6e945 is 50, key is test_row_0/B:col10/1733201142837/Put/seqid=0 2024-12-03T04:45:42,922 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/5434d60aedf0463eb83e9c60bf59f880 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/5434d60aedf0463eb83e9c60bf59f880 2024-12-03T04:45:42,927 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/A of 5585179eb550a7f04eee9f770b5bea8c into 5434d60aedf0463eb83e9c60bf59f880(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:42,927 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:42,927 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/A, priority=11, startTime=1733201142834; duration=0sec 2024-12-03T04:45:42,927 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:42,927 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:A 2024-12-03T04:45:42,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742447_1623 (size=12301) 2024-12-03T04:45:42,933 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#C#compaction#527 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:42,934 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/785d5af5b7b3454ab33a1765723aebc8 is 50, key is test_row_0/C:col10/1733201142065/Put/seqid=0 2024-12-03T04:45:42,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/7e293da106a545139ed4d5c142d6e945 2024-12-03T04:45:42,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742448_1624 (size=12847) 2024-12-03T04:45:42,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/cdf597b7195c499da0d15b4b987e4979 is 50, key is test_row_0/C:col10/1733201142837/Put/seqid=0 2024-12-03T04:45:42,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742449_1625 (size=12301) 2024-12-03T04:45:42,948 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/785d5af5b7b3454ab33a1765723aebc8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/785d5af5b7b3454ab33a1765723aebc8 2024-12-03T04:45:42,956 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/C of 5585179eb550a7f04eee9f770b5bea8c into 785d5af5b7b3454ab33a1765723aebc8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:42,956 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:42,956 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/C, priority=11, startTime=1733201142834; duration=0sec 2024-12-03T04:45:42,956 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:42,956 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:C 2024-12-03T04:45:43,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201203021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:43,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T04:45:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:43,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T04:45:43,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T04:45:43,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:43,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:43,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:43,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:43,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:43,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201203232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:43,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/cdf597b7195c499da0d15b4b987e4979 2024-12-03T04:45:43,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/7ec145b221af4961b9159457e5049c42 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/7ec145b221af4961b9159457e5049c42 2024-12-03T04:45:43,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/7ec145b221af4961b9159457e5049c42, entries=200, sequenceid=297, filesize=39.0 K 2024-12-03T04:45:43,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T04:45:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:43,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/7e293da106a545139ed4d5c142d6e945 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7e293da106a545139ed4d5c142d6e945 2024-12-03T04:45:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:43,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:43,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7e293da106a545139ed4d5c142d6e945, entries=150, sequenceid=297, filesize=12.0 K 2024-12-03T04:45:43,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/cdf597b7195c499da0d15b4b987e4979 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cdf597b7195c499da0d15b4b987e4979 2024-12-03T04:45:43,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cdf597b7195c499da0d15b4b987e4979, entries=150, sequenceid=297, filesize=12.0 K 2024-12-03T04:45:43,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5585179eb550a7f04eee9f770b5bea8c in 525ms, sequenceid=297, compaction requested=false 2024-12-03T04:45:43,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T04:45:43,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T04:45:43,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:43,509 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T04:45:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:43,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:43,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031313ec04584647d29ebe1cc1d68943ef_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201142890/Put/seqid=0 2024-12-03T04:45:43,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742450_1626 (size=12454) 2024-12-03T04:45:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:43,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:43,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201203629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:43,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201203742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T04:45:43,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:43,924 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031313ec04584647d29ebe1cc1d68943ef_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031313ec04584647d29ebe1cc1d68943ef_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:43,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/3afea7fb25e54c0194e2f3781bce6d50, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/3afea7fb25e54c0194e2f3781bce6d50 is 175, key is test_row_0/A:col10/1733201142890/Put/seqid=0 2024-12-03T04:45:43,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742451_1627 (size=31255) 2024-12-03T04:45:43,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201203952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:44,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201204261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:44,329 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/3afea7fb25e54c0194e2f3781bce6d50 2024-12-03T04:45:44,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/652595d8c94344c083b8af9dcc3126a8 is 50, key is test_row_0/B:col10/1733201142890/Put/seqid=0 2024-12-03T04:45:44,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742452_1628 (size=12301) 2024-12-03T04:45:44,747 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/652595d8c94344c083b8af9dcc3126a8 2024-12-03T04:45:44,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/83058fa2ff3e4f2bb524e8a87f8555d6 is 50, key is test_row_0/C:col10/1733201142890/Put/seqid=0 2024-12-03T04:45:44,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742453_1629 (size=12301) 2024-12-03T04:45:44,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201204771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T04:45:45,167 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/83058fa2ff3e4f2bb524e8a87f8555d6 2024-12-03T04:45:45,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/3afea7fb25e54c0194e2f3781bce6d50 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/3afea7fb25e54c0194e2f3781bce6d50 2024-12-03T04:45:45,173 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/3afea7fb25e54c0194e2f3781bce6d50, entries=150, sequenceid=313, filesize=30.5 K 2024-12-03T04:45:45,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/652595d8c94344c083b8af9dcc3126a8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/652595d8c94344c083b8af9dcc3126a8 2024-12-03T04:45:45,177 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/652595d8c94344c083b8af9dcc3126a8, entries=150, sequenceid=313, filesize=12.0 K 2024-12-03T04:45:45,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/83058fa2ff3e4f2bb524e8a87f8555d6 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/83058fa2ff3e4f2bb524e8a87f8555d6 2024-12-03T04:45:45,180 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/83058fa2ff3e4f2bb524e8a87f8555d6, entries=150, sequenceid=313, filesize=12.0 K 2024-12-03T04:45:45,181 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5585179eb550a7f04eee9f770b5bea8c in 1671ms, sequenceid=313, compaction requested=true 2024-12-03T04:45:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-03T04:45:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-03T04:45:45,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-03T04:45:45,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4370 sec 2024-12-03T04:45:45,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.4420 sec 2024-12-03T04:45:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:45,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T04:45:45,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:45,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:45,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:45,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:45,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:45,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:45,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba4f809c74f94da09f232ed040b3d531_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201143600/Put/seqid=0 2024-12-03T04:45:45,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742454_1630 (size=14994) 2024-12-03T04:45:45,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201205834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:45,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:45,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:45,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:45,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201205941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:46,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201206152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:46,207 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:46,210 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba4f809c74f94da09f232ed040b3d531_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba4f809c74f94da09f232ed040b3d531_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:46,211 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0d57150e768d47668001bdfc5ebf0133, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:46,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0d57150e768d47668001bdfc5ebf0133 is 175, key is test_row_0/A:col10/1733201143600/Put/seqid=0 2024-12-03T04:45:46,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742455_1631 (size=39949) 2024-12-03T04:45:46,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201206461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:46,617 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0d57150e768d47668001bdfc5ebf0133 2024-12-03T04:45:46,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/3fb94b6eb114467991aa6e5b1708faa3 is 50, key is test_row_0/B:col10/1733201143600/Put/seqid=0 2024-12-03T04:45:46,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742456_1632 (size=12301) 2024-12-03T04:45:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T04:45:46,912 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:46,913 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T04:45:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-03T04:45:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T04:45:46,915 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T04:45:46,916 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T04:45:46,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T04:45:46,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-03T04:45:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43193 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44426 deadline: 1733201206974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 2024-12-03T04:45:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 , the old value is region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T04:45:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., hostname=32beccd76b35,43193,1733200986730, seqNum=5 because the exception is null or not the one we care about 2024-12-03T04:45:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T04:45:47,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T04:45:47,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:47,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:47,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:47,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/3fb94b6eb114467991aa6e5b1708faa3 2024-12-03T04:45:47,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T04:45:47,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/9e19aef9504741e4a38e2be4afe385f8 is 50, key is test_row_0/C:col10/1733201143600/Put/seqid=0 2024-12-03T04:45:47,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742457_1633 (size=12301) 2024-12-03T04:45:47,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/9e19aef9504741e4a38e2be4afe385f8 2024-12-03T04:45:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/0d57150e768d47668001bdfc5ebf0133 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0d57150e768d47668001bdfc5ebf0133 2024-12-03T04:45:47,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0d57150e768d47668001bdfc5ebf0133, entries=200, sequenceid=333, filesize=39.0 K 2024-12-03T04:45:47,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/3fb94b6eb114467991aa6e5b1708faa3 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/3fb94b6eb114467991aa6e5b1708faa3 2024-12-03T04:45:47,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/3fb94b6eb114467991aa6e5b1708faa3, entries=150, sequenceid=333, filesize=12.0 K 2024-12-03T04:45:47,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/9e19aef9504741e4a38e2be4afe385f8 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/9e19aef9504741e4a38e2be4afe385f8 2024-12-03T04:45:47,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/9e19aef9504741e4a38e2be4afe385f8, entries=150, sequenceid=333, filesize=12.0 K 2024-12-03T04:45:47,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5585179eb550a7f04eee9f770b5bea8c in 1366ms, sequenceid=333, compaction requested=true 2024-12-03T04:45:47,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:47,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T04:45:47,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:47,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T04:45:47,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:47,159 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:47,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5585179eb550a7f04eee9f770b5bea8c:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T04:45:47,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:47,159 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:47,161 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:47,161 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143061 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:47,161 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/A is initiating minor compaction (all files) 2024-12-03T04:45:47,161 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/B is initiating minor compaction (all files) 2024-12-03T04:45:47,161 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/A in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:47,161 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/B in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:47,162 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/5434d60aedf0463eb83e9c60bf59f880, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/7ec145b221af4961b9159457e5049c42, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/3afea7fb25e54c0194e2f3781bce6d50, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0d57150e768d47668001bdfc5ebf0133] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=139.7 K 2024-12-03T04:45:47,162 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/a70f524773fa4e4bafbc80435d9aacec, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7e293da106a545139ed4d5c142d6e945, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/652595d8c94344c083b8af9dcc3126a8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/3fb94b6eb114467991aa6e5b1708faa3] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=48.6 K 2024-12-03T04:45:47,162 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:47,162 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. files: [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/5434d60aedf0463eb83e9c60bf59f880, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/7ec145b221af4961b9159457e5049c42, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/3afea7fb25e54c0194e2f3781bce6d50, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0d57150e768d47668001bdfc5ebf0133] 2024-12-03T04:45:47,163 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting a70f524773fa4e4bafbc80435d9aacec, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733201140839 2024-12-03T04:45:47,163 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5434d60aedf0463eb83e9c60bf59f880, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733201140839 2024-12-03T04:45:47,163 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e293da106a545139ed4d5c142d6e945, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733201142788 2024-12-03T04:45:47,163 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ec145b221af4961b9159457e5049c42, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733201142788 2024-12-03T04:45:47,163 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3afea7fb25e54c0194e2f3781bce6d50, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733201142850 2024-12-03T04:45:47,164 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 652595d8c94344c083b8af9dcc3126a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733201142850 2024-12-03T04:45:47,164 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d57150e768d47668001bdfc5ebf0133, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733201143600 2024-12-03T04:45:47,164 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fb94b6eb114467991aa6e5b1708faa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733201143600 2024-12-03T04:45:47,182 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:47,192 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#B#compaction#536 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:47,193 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/f657bfc230ad4fd88806bc007bd0a11a is 50, key is test_row_0/B:col10/1733201143600/Put/seqid=0 2024-12-03T04:45:47,193 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203c023b00108f24dd297f348db7ac9c4bc_5585179eb550a7f04eee9f770b5bea8c store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:47,196 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203c023b00108f24dd297f348db7ac9c4bc_5585179eb550a7f04eee9f770b5bea8c, store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:47,196 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c023b00108f24dd297f348db7ac9c4bc_5585179eb550a7f04eee9f770b5bea8c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:47,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43193 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T04:45:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:47,229 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T04:45:47,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:47,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:47,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:47,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:47,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:47,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T04:45:47,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742458_1634 (size=12983) 2024-12-03T04:45:47,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742459_1635 (size=4469) 2024-12-03T04:45:47,253 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/f657bfc230ad4fd88806bc007bd0a11a as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/f657bfc230ad4fd88806bc007bd0a11a 2024-12-03T04:45:47,267 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/B of 5585179eb550a7f04eee9f770b5bea8c into f657bfc230ad4fd88806bc007bd0a11a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:47,267 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:47,267 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/B, priority=12, startTime=1733201147159; duration=0sec 2024-12-03T04:45:47,267 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T04:45:47,267 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:B 2024-12-03T04:45:47,268 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T04:45:47,270 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T04:45:47,270 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1541): 5585179eb550a7f04eee9f770b5bea8c/C is initiating minor compaction (all files) 2024-12-03T04:45:47,270 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5585179eb550a7f04eee9f770b5bea8c/C in TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:47,270 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/785d5af5b7b3454ab33a1765723aebc8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cdf597b7195c499da0d15b4b987e4979, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/83058fa2ff3e4f2bb524e8a87f8555d6, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/9e19aef9504741e4a38e2be4afe385f8] into tmpdir=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp, totalSize=48.6 K 2024-12-03T04:45:47,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a800c0162eda462bacbf44c3ff261348_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201145809/Put/seqid=0 2024-12-03T04:45:47,270 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 785d5af5b7b3454ab33a1765723aebc8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733201140839 2024-12-03T04:45:47,271 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting cdf597b7195c499da0d15b4b987e4979, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733201142788 2024-12-03T04:45:47,272 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 83058fa2ff3e4f2bb524e8a87f8555d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733201142850 2024-12-03T04:45:47,272 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e19aef9504741e4a38e2be4afe385f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733201143600 2024-12-03T04:45:47,285 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#C#compaction#538 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:47,286 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/ef50cc5d0d9945f8ad55cf66278caf24 is 50, key is test_row_0/C:col10/1733201143600/Put/seqid=0 2024-12-03T04:45:47,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742460_1636 (size=12454) 2024-12-03T04:45:47,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:47,321 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a800c0162eda462bacbf44c3ff261348_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a800c0162eda462bacbf44c3ff261348_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:47,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/8f78232f6eea4478ba1457ed79926489, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/8f78232f6eea4478ba1457ed79926489 is 175, key is test_row_0/A:col10/1733201145809/Put/seqid=0 2024-12-03T04:45:47,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742461_1637 (size=12983) 2024-12-03T04:45:47,335 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/ef50cc5d0d9945f8ad55cf66278caf24 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/ef50cc5d0d9945f8ad55cf66278caf24 2024-12-03T04:45:47,339 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/C of 5585179eb550a7f04eee9f770b5bea8c into ef50cc5d0d9945f8ad55cf66278caf24(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:47,339 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:47,339 INFO [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/C, priority=12, startTime=1733201147159; duration=0sec 2024-12-03T04:45:47,339 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:47,339 DEBUG [RS:0;32beccd76b35:43193-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:C 2024-12-03T04:45:47,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742462_1638 (size=31255) 2024-12-03T04:45:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T04:45:47,669 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5585179eb550a7f04eee9f770b5bea8c#A#compaction#535 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-12-03T04:45:47,670 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/5233c7b909b34fb6b55d102817c26860 is 175, key is test_row_0/A:col10/1733201143600/Put/seqid=0 2024-12-03T04:45:47,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742463_1639 (size=31937) 2024-12-03T04:45:47,716 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/5233c7b909b34fb6b55d102817c26860 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/5233c7b909b34fb6b55d102817c26860 2024-12-03T04:45:47,727 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5585179eb550a7f04eee9f770b5bea8c/A of 5585179eb550a7f04eee9f770b5bea8c into 5233c7b909b34fb6b55d102817c26860(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T04:45:47,727 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:47,727 INFO [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., storeName=5585179eb550a7f04eee9f770b5bea8c/A, priority=12, startTime=1733201147159; duration=0sec 2024-12-03T04:45:47,727 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T04:45:47,727 DEBUG [RS:0;32beccd76b35:43193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5585179eb550a7f04eee9f770b5bea8c:A 2024-12-03T04:45:47,773 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/8f78232f6eea4478ba1457ed79926489 2024-12-03T04:45:47,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/fab2ec3df20949c3a40e531a5d63f979 is 50, key is test_row_0/B:col10/1733201145809/Put/seqid=0 2024-12-03T04:45:47,823 INFO [Thread-2472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2472. 2024-12-03T04:45:47,823 DEBUG [Thread-2472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:47,823 DEBUG [Thread-2472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,824 INFO [Thread-2466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2466. 2024-12-03T04:45:47,824 DEBUG [Thread-2466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:47,824 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,824 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:47,824 DEBUG [Thread-2472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:47,826 INFO [Thread-2464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2464. 2024-12-03T04:45:47,826 DEBUG [Thread-2464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:47,826 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,826 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,826 INFO [Thread-2470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2470. 2024-12-03T04:45:47,826 DEBUG [Thread-2470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:47,826 DEBUG [Thread-2470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,826 DEBUG [Thread-2470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:47,828 INFO [Thread-2468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2468. 2024-12-03T04:45:47,828 DEBUG [Thread-2468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:47,828 DEBUG [Thread-2468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,828 DEBUG [Thread-2468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:47,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:47,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742464_1640 (size=12301) 2024-12-03T04:45:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43193 {}] regionserver.HRegion(8855): Flush requested on 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:47,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. as already flushing 2024-12-03T04:45:47,993 INFO [Thread-2461 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2461. 2024-12-03T04:45:47,993 DEBUG [Thread-2461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:47,993 DEBUG [Thread-2461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,994 DEBUG [Thread-2461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:47,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T04:45:48,239 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/fab2ec3df20949c3a40e531a5d63f979 2024-12-03T04:45:48,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/dd4743f8826d4289ba703705f243feea is 50, key is test_row_0/C:col10/1733201145809/Put/seqid=0 2024-12-03T04:45:48,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742465_1641 (size=12301) 2024-12-03T04:45:48,650 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/dd4743f8826d4289ba703705f243feea 2024-12-03T04:45:48,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/8f78232f6eea4478ba1457ed79926489 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/8f78232f6eea4478ba1457ed79926489 2024-12-03T04:45:48,655 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/8f78232f6eea4478ba1457ed79926489, entries=150, sequenceid=345, filesize=30.5 K 2024-12-03T04:45:48,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/fab2ec3df20949c3a40e531a5d63f979 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/fab2ec3df20949c3a40e531a5d63f979 2024-12-03T04:45:48,658 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/fab2ec3df20949c3a40e531a5d63f979, entries=150, sequenceid=345, filesize=12.0 K 2024-12-03T04:45:48,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/dd4743f8826d4289ba703705f243feea as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/dd4743f8826d4289ba703705f243feea 2024-12-03T04:45:48,662 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/dd4743f8826d4289ba703705f243feea, entries=150, sequenceid=345, filesize=12.0 K 2024-12-03T04:45:48,662 INFO [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=6.71 KB/6870 for 5585179eb550a7f04eee9f770b5bea8c in 1433ms, sequenceid=345, compaction requested=false 2024-12-03T04:45:48,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 5585179eb550a7f04eee9f770b5bea8c: 2024-12-03T04:45:48,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:48,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32beccd76b35:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-03T04:45:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-03T04:45:48,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-03T04:45:48,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7480 sec 2024-12-03T04:45:48,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.7520 sec 2024-12-03T04:45:48,933 INFO [Thread-2455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2455. 2024-12-03T04:45:48,933 INFO [Thread-2457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2457. 2024-12-03T04:45:48,933 DEBUG [Thread-2457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:48,933 DEBUG [Thread-2455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:48,933 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:48,933 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:48,933 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:48,933 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:48,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:48,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:48,982 INFO [Thread-2453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2453. 2024-12-03T04:45:48,982 DEBUG [Thread-2453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:48,982 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:48,982 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:48,982 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:48,992 INFO [Thread-2459 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2459. 2024-12-03T04:45:48,992 DEBUG [Thread-2459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T04:45:48,992 DEBUG [Thread-2459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:48,992 DEBUG [Thread-2459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:48,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T04:45:49,072 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 221 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2760 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2554 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2610 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2775 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2580 2024-12-03T04:45:49,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T04:45:49,072 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T04:45:49,073 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T04:45:49,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:49,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:49,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:49,073 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T04:45:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T04:45:49,076 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201149076"}]},"ts":"1733201149076"} 2024-12-03T04:45:49,077 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T04:45:49,077 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T04:45:49,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T04:45:49,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, UNASSIGN}] 2024-12-03T04:45:49,080 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, UNASSIGN 2024-12-03T04:45:49,080 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=5585179eb550a7f04eee9f770b5bea8c, regionState=CLOSING, regionLocation=32beccd76b35,43193,1733200986730 2024-12-03T04:45:49,081 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38397 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=32beccd76b35,43193,1733200986730, table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T04:45:49,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, UNASSIGN because future has completed 2024-12-03T04:45:49,082 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T04:45:49,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730}] 2024-12-03T04:45:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T04:45:49,234 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:49,234 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T04:45:49,234 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing 5585179eb550a7f04eee9f770b5bea8c, disabling compactions & flushes 2024-12-03T04:45:49,234 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:49,234 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:49,234 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. after waiting 0 ms 2024-12-03T04:45:49,234 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:49,234 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing 5585179eb550a7f04eee9f770b5bea8c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T04:45:49,235 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=A 2024-12-03T04:45:49,235 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:49,235 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=B 2024-12-03T04:45:49,235 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:49,235 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5585179eb550a7f04eee9f770b5bea8c, store=C 2024-12-03T04:45:49,235 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T04:45:49,239 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203aa524ce2dc8c4877a67e56b687b9d972_5585179eb550a7f04eee9f770b5bea8c is 50, key is test_row_0/A:col10/1733201148932/Put/seqid=0 2024-12-03T04:45:49,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742466_1642 (size=9914) 2024-12-03T04:45:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T04:45:49,643 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T04:45:49,645 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203aa524ce2dc8c4877a67e56b687b9d972_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aa524ce2dc8c4877a67e56b687b9d972_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:49,646 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/a08960f067324539be4cb4f3f5cb84ba, store: [table=TestAcidGuarantees family=A region=5585179eb550a7f04eee9f770b5bea8c] 2024-12-03T04:45:49,646 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/a08960f067324539be4cb4f3f5cb84ba is 175, key is test_row_0/A:col10/1733201148932/Put/seqid=0 2024-12-03T04:45:49,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742467_1643 (size=22561) 2024-12-03T04:45:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T04:45:50,050 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/a08960f067324539be4cb4f3f5cb84ba 2024-12-03T04:45:50,055 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/693b740d53514cdea02b048874da5fb2 is 50, key is test_row_0/B:col10/1733201148932/Put/seqid=0 2024-12-03T04:45:50,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742468_1644 (size=9857) 2024-12-03T04:45:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T04:45:50,458 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/693b740d53514cdea02b048874da5fb2 2024-12-03T04:45:50,464 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/7e9f98c8e0694670a727264c80a9c201 is 50, key is test_row_0/C:col10/1733201148932/Put/seqid=0 2024-12-03T04:45:50,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742469_1645 (size=9857) 2024-12-03T04:45:50,867 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/7e9f98c8e0694670a727264c80a9c201 2024-12-03T04:45:50,873 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/A/a08960f067324539be4cb4f3f5cb84ba as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/a08960f067324539be4cb4f3f5cb84ba 2024-12-03T04:45:50,877 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/a08960f067324539be4cb4f3f5cb84ba, entries=100, sequenceid=356, filesize=22.0 K 2024-12-03T04:45:50,878 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/B/693b740d53514cdea02b048874da5fb2 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/693b740d53514cdea02b048874da5fb2 2024-12-03T04:45:50,881 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/693b740d53514cdea02b048874da5fb2, entries=100, sequenceid=356, filesize=9.6 K 2024-12-03T04:45:50,888 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/.tmp/C/7e9f98c8e0694670a727264c80a9c201 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/7e9f98c8e0694670a727264c80a9c201 2024-12-03T04:45:50,896 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/7e9f98c8e0694670a727264c80a9c201, entries=100, sequenceid=356, filesize=9.6 K 2024-12-03T04:45:50,897 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5585179eb550a7f04eee9f770b5bea8c in 1662ms, sequenceid=356, compaction requested=true 2024-12-03T04:45:50,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/2855e8634d704be19842f3448fb76950, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/dbd474ee2b454ee08bc445f679a1d412, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d4fe66ed4c3747ff98387575db536731, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0b4078d42d0b4c04a94fe20ee5679f2d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/e0251da0987447b4a71e4090738f5830, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/4a70bacba16b443d997c0be57ce460da, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/c39778dcc3f84f5ea1f37999cae1a6bd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/55c235c4c285488e8f2f45f144c99da0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/71830851771f46ada9f78ebaa68d5c5a, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d973ae082a49479786b41c2a0ac6d41c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/de2be6e8cee84ee8a9f71f32c48d4345, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0a69e4822156465bad7ecf0f03e811fc, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/22491b9e703d4b2db988ae5172c72199, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/96a167988b8945298e974c9b973b2034, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/729ddfaead64477abf85e460ecac5d13, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/5434d60aedf0463eb83e9c60bf59f880, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/b1c4caf98e8b4cce89cfb2a0b49ad92f, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/7ec145b221af4961b9159457e5049c42, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/3afea7fb25e54c0194e2f3781bce6d50, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0d57150e768d47668001bdfc5ebf0133] to archive 2024-12-03T04:45:50,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:45:50,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/2855e8634d704be19842f3448fb76950 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/2855e8634d704be19842f3448fb76950 2024-12-03T04:45:50,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/dbd474ee2b454ee08bc445f679a1d412 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/dbd474ee2b454ee08bc445f679a1d412 2024-12-03T04:45:50,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d4fe66ed4c3747ff98387575db536731 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d4fe66ed4c3747ff98387575db536731 2024-12-03T04:45:50,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0b4078d42d0b4c04a94fe20ee5679f2d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0b4078d42d0b4c04a94fe20ee5679f2d 2024-12-03T04:45:50,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/e0251da0987447b4a71e4090738f5830 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/e0251da0987447b4a71e4090738f5830 2024-12-03T04:45:50,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/4a70bacba16b443d997c0be57ce460da to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/4a70bacba16b443d997c0be57ce460da 2024-12-03T04:45:50,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/c39778dcc3f84f5ea1f37999cae1a6bd to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/c39778dcc3f84f5ea1f37999cae1a6bd 2024-12-03T04:45:50,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/55c235c4c285488e8f2f45f144c99da0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/55c235c4c285488e8f2f45f144c99da0 2024-12-03T04:45:50,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/71830851771f46ada9f78ebaa68d5c5a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/71830851771f46ada9f78ebaa68d5c5a 2024-12-03T04:45:50,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d973ae082a49479786b41c2a0ac6d41c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/d973ae082a49479786b41c2a0ac6d41c 2024-12-03T04:45:50,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/de2be6e8cee84ee8a9f71f32c48d4345 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/de2be6e8cee84ee8a9f71f32c48d4345 2024-12-03T04:45:50,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0a69e4822156465bad7ecf0f03e811fc to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0a69e4822156465bad7ecf0f03e811fc 2024-12-03T04:45:50,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/22491b9e703d4b2db988ae5172c72199 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/22491b9e703d4b2db988ae5172c72199 2024-12-03T04:45:50,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/96a167988b8945298e974c9b973b2034 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/96a167988b8945298e974c9b973b2034 2024-12-03T04:45:50,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/729ddfaead64477abf85e460ecac5d13 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/729ddfaead64477abf85e460ecac5d13 2024-12-03T04:45:50,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/5434d60aedf0463eb83e9c60bf59f880 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/5434d60aedf0463eb83e9c60bf59f880 2024-12-03T04:45:50,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/b1c4caf98e8b4cce89cfb2a0b49ad92f to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/b1c4caf98e8b4cce89cfb2a0b49ad92f 2024-12-03T04:45:50,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/7ec145b221af4961b9159457e5049c42 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/7ec145b221af4961b9159457e5049c42 2024-12-03T04:45:50,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/3afea7fb25e54c0194e2f3781bce6d50 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/3afea7fb25e54c0194e2f3781bce6d50 2024-12-03T04:45:50,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0d57150e768d47668001bdfc5ebf0133 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/0d57150e768d47668001bdfc5ebf0133 2024-12-03T04:45:50,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7743445d60f1448fabf8272830b447eb, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/fc3b98ca33bc48d1997d4df27bff61ac, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/3755ffae8bcb45c0b48054c865d749b5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/272dc9c39b8d48b999f1328b9564d845, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/1530d89c8ce24d34aca979783ba9f649, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/a7f833bb1cc147e9af6017ac1c3fca8e, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/41857dfd2d6c4fffbd439fb8c58f0722, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/d9845a1b7ff3452d8cf5de4f7a46efe4, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/06db24845e1a4ecaa42a33d6b11f3fd5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/92becd190a9e40d2a6619f6b1b3ab016, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/82e23d1b599f4c209d51ad7e8287fc2d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/6c850a1dcb344fbdb826fd3765ff91d0, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/b2f71e3c2d18408ea459942c201d0b95, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/07b52e1b247a4dd2bb18868f2763ca3c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/86ceeccfab9f4151ae68836b6491476d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/a70f524773fa4e4bafbc80435d9aacec, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/c9810086290043b99ba7158b77464a8d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7e293da106a545139ed4d5c142d6e945, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/652595d8c94344c083b8af9dcc3126a8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/3fb94b6eb114467991aa6e5b1708faa3] to archive 2024-12-03T04:45:50,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:45:50,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7743445d60f1448fabf8272830b447eb to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7743445d60f1448fabf8272830b447eb 2024-12-03T04:45:50,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/fc3b98ca33bc48d1997d4df27bff61ac to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/fc3b98ca33bc48d1997d4df27bff61ac 2024-12-03T04:45:50,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/3755ffae8bcb45c0b48054c865d749b5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/3755ffae8bcb45c0b48054c865d749b5 2024-12-03T04:45:50,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/272dc9c39b8d48b999f1328b9564d845 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/272dc9c39b8d48b999f1328b9564d845 2024-12-03T04:45:50,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/1530d89c8ce24d34aca979783ba9f649 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/1530d89c8ce24d34aca979783ba9f649 2024-12-03T04:45:50,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/a7f833bb1cc147e9af6017ac1c3fca8e to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/a7f833bb1cc147e9af6017ac1c3fca8e 2024-12-03T04:45:50,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/41857dfd2d6c4fffbd439fb8c58f0722 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/41857dfd2d6c4fffbd439fb8c58f0722 2024-12-03T04:45:50,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/d9845a1b7ff3452d8cf5de4f7a46efe4 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/d9845a1b7ff3452d8cf5de4f7a46efe4 2024-12-03T04:45:50,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/06db24845e1a4ecaa42a33d6b11f3fd5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/06db24845e1a4ecaa42a33d6b11f3fd5 2024-12-03T04:45:50,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/92becd190a9e40d2a6619f6b1b3ab016 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/92becd190a9e40d2a6619f6b1b3ab016 2024-12-03T04:45:50,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/82e23d1b599f4c209d51ad7e8287fc2d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/82e23d1b599f4c209d51ad7e8287fc2d 2024-12-03T04:45:50,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/6c850a1dcb344fbdb826fd3765ff91d0 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/6c850a1dcb344fbdb826fd3765ff91d0 2024-12-03T04:45:50,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/b2f71e3c2d18408ea459942c201d0b95 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/b2f71e3c2d18408ea459942c201d0b95 2024-12-03T04:45:50,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/07b52e1b247a4dd2bb18868f2763ca3c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/07b52e1b247a4dd2bb18868f2763ca3c 2024-12-03T04:45:50,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/86ceeccfab9f4151ae68836b6491476d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/86ceeccfab9f4151ae68836b6491476d 2024-12-03T04:45:50,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/a70f524773fa4e4bafbc80435d9aacec to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/a70f524773fa4e4bafbc80435d9aacec 2024-12-03T04:45:50,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/c9810086290043b99ba7158b77464a8d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/c9810086290043b99ba7158b77464a8d 2024-12-03T04:45:50,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7e293da106a545139ed4d5c142d6e945 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/7e293da106a545139ed4d5c142d6e945 2024-12-03T04:45:50,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/652595d8c94344c083b8af9dcc3126a8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/652595d8c94344c083b8af9dcc3126a8 2024-12-03T04:45:50,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/3fb94b6eb114467991aa6e5b1708faa3 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/3fb94b6eb114467991aa6e5b1708faa3 2024-12-03T04:45:50,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/77199354f2344eb2adb943faa54e0f09, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/52999967c95b4ee782097913d44ab179, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/b343e4ec3a9747c5b827fb012f60eeaa, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/5d1fb28e140f4035a9add5de1dcdc6b5, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/2769ea71f0d64aefbafe33702951ffa2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/3e8f06b733954cb089206e95c6f79f02, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/9c95c58ba69748ca823c955857ba6b9c, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/1d805bace17a4f24a76c6f656ae4f731, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/025c750a9545445cb78c496d6976c5a2, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/f2142837f53c4542bc6e6b5c42c8334b, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/348043d7090d49f28ab940aa982648ab, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/efc8b9771cf7470091f37e910dc356dd, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cc2ff7c66dc04159b3c66d012bbdb0ee, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/03f4aac4165747359e6f0b9dbcebd965, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/902d0e67d81840f6821f199ca9a01a80, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/785d5af5b7b3454ab33a1765723aebc8, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/08be618f9a5449df9f2a9d2799fb417d, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cdf597b7195c499da0d15b4b987e4979, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/83058fa2ff3e4f2bb524e8a87f8555d6, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/9e19aef9504741e4a38e2be4afe385f8] to archive 2024-12-03T04:45:50,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T04:45:51,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/77199354f2344eb2adb943faa54e0f09 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/77199354f2344eb2adb943faa54e0f09 2024-12-03T04:45:51,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/52999967c95b4ee782097913d44ab179 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/52999967c95b4ee782097913d44ab179 2024-12-03T04:45:51,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/b343e4ec3a9747c5b827fb012f60eeaa to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/b343e4ec3a9747c5b827fb012f60eeaa 2024-12-03T04:45:51,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/5d1fb28e140f4035a9add5de1dcdc6b5 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/5d1fb28e140f4035a9add5de1dcdc6b5 2024-12-03T04:45:51,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/2769ea71f0d64aefbafe33702951ffa2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/2769ea71f0d64aefbafe33702951ffa2 2024-12-03T04:45:51,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/3e8f06b733954cb089206e95c6f79f02 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/3e8f06b733954cb089206e95c6f79f02 2024-12-03T04:45:51,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/9c95c58ba69748ca823c955857ba6b9c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/9c95c58ba69748ca823c955857ba6b9c 2024-12-03T04:45:51,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/1d805bace17a4f24a76c6f656ae4f731 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/1d805bace17a4f24a76c6f656ae4f731 2024-12-03T04:45:51,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/025c750a9545445cb78c496d6976c5a2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/025c750a9545445cb78c496d6976c5a2 2024-12-03T04:45:51,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/f2142837f53c4542bc6e6b5c42c8334b to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/f2142837f53c4542bc6e6b5c42c8334b 2024-12-03T04:45:51,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/348043d7090d49f28ab940aa982648ab to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/348043d7090d49f28ab940aa982648ab 2024-12-03T04:45:51,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/efc8b9771cf7470091f37e910dc356dd to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/efc8b9771cf7470091f37e910dc356dd 2024-12-03T04:45:51,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cc2ff7c66dc04159b3c66d012bbdb0ee to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cc2ff7c66dc04159b3c66d012bbdb0ee 2024-12-03T04:45:51,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/03f4aac4165747359e6f0b9dbcebd965 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/03f4aac4165747359e6f0b9dbcebd965 2024-12-03T04:45:51,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/902d0e67d81840f6821f199ca9a01a80 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/902d0e67d81840f6821f199ca9a01a80 2024-12-03T04:45:51,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/785d5af5b7b3454ab33a1765723aebc8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/785d5af5b7b3454ab33a1765723aebc8 2024-12-03T04:45:51,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/08be618f9a5449df9f2a9d2799fb417d to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/08be618f9a5449df9f2a9d2799fb417d 2024-12-03T04:45:51,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cdf597b7195c499da0d15b4b987e4979 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/cdf597b7195c499da0d15b4b987e4979 2024-12-03T04:45:51,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/83058fa2ff3e4f2bb524e8a87f8555d6 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/83058fa2ff3e4f2bb524e8a87f8555d6 2024-12-03T04:45:51,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/9e19aef9504741e4a38e2be4afe385f8 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/9e19aef9504741e4a38e2be4afe385f8 2024-12-03T04:45:51,046 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/recovered.edits/359.seqid, newMaxSeqId=359, maxSeqId=4 2024-12-03T04:45:51,047 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. 2024-12-03T04:45:51,047 DEBUG [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for 5585179eb550a7f04eee9f770b5bea8c: Waiting for close lock at 1733201149234Running coprocessor pre-close hooks at 1733201149234Disabling compacts and flushes for region at 1733201149234Disabling writes for close at 1733201149234Obtaining lock to block concurrent updates at 1733201149234Preparing flush snapshotting stores in 5585179eb550a7f04eee9f770b5bea8c at 1733201149234Finished memstore snapshotting TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733201149235 (+1 ms)Flushing stores of TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c. at 1733201149235Flushing 5585179eb550a7f04eee9f770b5bea8c/A: creating writer at 1733201149235Flushing mob file 5585179eb550a7f04eee9f770b5bea8c/A: appending metadata at 1733201149239 (+4 ms)Flushing mob file 5585179eb550a7f04eee9f770b5bea8c/A: closing flushed file at 1733201149239Flushing 5585179eb550a7f04eee9f770b5bea8c/A: appending metadata at 1733201149646 (+407 ms)Flushing 5585179eb550a7f04eee9f770b5bea8c/A: closing flushed file at 1733201149646Flushing 5585179eb550a7f04eee9f770b5bea8c/B: creating writer at 1733201150052 (+406 ms)Flushing 5585179eb550a7f04eee9f770b5bea8c/B: appending metadata at 1733201150054 (+2 ms)Flushing 5585179eb550a7f04eee9f770b5bea8c/B: closing flushed file at 1733201150054Flushing 5585179eb550a7f04eee9f770b5bea8c/C: creating writer at 1733201150461 (+407 ms)Flushing 5585179eb550a7f04eee9f770b5bea8c/C: appending metadata at 1733201150463 (+2 ms)Flushing 5585179eb550a7f04eee9f770b5bea8c/C: closing flushed file at 1733201150463Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49325ab1: reopening flushed file at 1733201150872 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ee07948: reopening flushed file at 1733201150878 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fff0bf4: reopening flushed file at 1733201150882 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5585179eb550a7f04eee9f770b5bea8c in 1662ms, sequenceid=356, compaction requested=true at 1733201150897 (+15 ms)Writing region close event to WAL at 1733201151039 (+142 ms)Running coprocessor post-close hooks at 1733201151047 (+8 ms)Closed at 1733201151047 2024-12-03T04:45:51,051 INFO [RS_CLOSE_REGION-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed 5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,051 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=5585179eb550a7f04eee9f770b5bea8c, regionState=CLOSED 2024-12-03T04:45:51,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 because future has completed 2024-12-03T04:45:51,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-03T04:45:51,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure 5585179eb550a7f04eee9f770b5bea8c, server=32beccd76b35,43193,1733200986730 in 1.9770 sec 2024-12-03T04:45:51,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-12-03T04:45:51,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5585179eb550a7f04eee9f770b5bea8c, UNASSIGN in 1.9830 sec 2024-12-03T04:45:51,068 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733201151068"}]},"ts":"1733201151068"} 2024-12-03T04:45:51,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-03T04:45:51,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9860 sec 2024-12-03T04:45:51,070 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T04:45:51,070 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T04:45:51,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9980 sec 2024-12-03T04:45:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T04:45:51,222 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:51,223 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T04:45:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:51,226 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:51,226 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T04:45:51,241 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,254 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C, FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/recovered.edits] 2024-12-03T04:45:51,261 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/5233c7b909b34fb6b55d102817c26860 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/5233c7b909b34fb6b55d102817c26860 2024-12-03T04:45:51,263 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/8f78232f6eea4478ba1457ed79926489 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/8f78232f6eea4478ba1457ed79926489 2024-12-03T04:45:51,266 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/a08960f067324539be4cb4f3f5cb84ba to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/A/a08960f067324539be4cb4f3f5cb84ba 2024-12-03T04:45:51,272 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/693b740d53514cdea02b048874da5fb2 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/693b740d53514cdea02b048874da5fb2 2024-12-03T04:45:51,273 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/f657bfc230ad4fd88806bc007bd0a11a to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/f657bfc230ad4fd88806bc007bd0a11a 2024-12-03T04:45:51,275 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/fab2ec3df20949c3a40e531a5d63f979 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/B/fab2ec3df20949c3a40e531a5d63f979 2024-12-03T04:45:51,277 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/7e9f98c8e0694670a727264c80a9c201 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/7e9f98c8e0694670a727264c80a9c201 2024-12-03T04:45:51,278 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/dd4743f8826d4289ba703705f243feea to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/dd4743f8826d4289ba703705f243feea 2024-12-03T04:45:51,280 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/ef50cc5d0d9945f8ad55cf66278caf24 to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/C/ef50cc5d0d9945f8ad55cf66278caf24 2024-12-03T04:45:51,283 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/recovered.edits/359.seqid to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c/recovered.edits/359.seqid 2024-12-03T04:45:51,284 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/default/TestAcidGuarantees/5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,284 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T04:45:51,286 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T04:45:51,287 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T04:45:51,293 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203057c27f74ec844359b082e5c9335285b_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203057c27f74ec844359b082e5c9335285b_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,298 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030b02087b07044f12a4c8f520c01fe0ea_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030b02087b07044f12a4c8f520c01fe0ea_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,306 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031313ec04584647d29ebe1cc1d68943ef_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031313ec04584647d29ebe1cc1d68943ef_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,310 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120318ecac6baed1438eba4aaedf4a8d4015_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120318ecac6baed1438eba4aaedf4a8d4015_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,313 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c957b8e3b52490db08ef7463854a16c_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c957b8e3b52490db08ef7463854a16c_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,315 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120360ccb16f545d45a1a0ec975d7293edd3_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120360ccb16f545d45a1a0ec975d7293edd3_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,317 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037d1481d915234d709b7e4b06c7a6c4ef_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037d1481d915234d709b7e4b06c7a6c4ef_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,318 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037f46f47c1d4a47a68f2f3e94a98a0230_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037f46f47c1d4a47a68f2f3e94a98a0230_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,320 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038460796d60b4476db2ecd57ecb0cf0a6_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038460796d60b4476db2ecd57ecb0cf0a6_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,322 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a20fb5ddb82c42d6ab39184e0d07ba69_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a20fb5ddb82c42d6ab39184e0d07ba69_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,323 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a800c0162eda462bacbf44c3ff261348_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a800c0162eda462bacbf44c3ff261348_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,327 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aa524ce2dc8c4877a67e56b687b9d972_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aa524ce2dc8c4877a67e56b687b9d972_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,328 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b0e0ba5811fb408c97d668eed30c9148_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b0e0ba5811fb408c97d668eed30c9148_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,332 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba4f809c74f94da09f232ed040b3d531_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba4f809c74f94da09f232ed040b3d531_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T04:45:51,333 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c8f4ae9cdb4a4da5a43c972bb337f64d_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c8f4ae9cdb4a4da5a43c972bb337f64d_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,337 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203caa35d073ee14265877153548ce0ed10_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203caa35d073ee14265877153548ce0ed10_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,340 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cb92f80dcf85409e98ecc108b2cfd037_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cb92f80dcf85409e98ecc108b2cfd037_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,342 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f6f81f3fdbaa45ac847a149fd1982033_5585179eb550a7f04eee9f770b5bea8c to hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f6f81f3fdbaa45ac847a149fd1982033_5585179eb550a7f04eee9f770b5bea8c 2024-12-03T04:45:51,343 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T04:45:51,345 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:51,349 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T04:45:51,352 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T04:45:51,354 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:51,354 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T04:45:51,355 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733201151354"}]},"ts":"9223372036854775807"} 2024-12-03T04:45:51,359 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T04:45:51,359 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5585179eb550a7f04eee9f770b5bea8c, NAME => 'TestAcidGuarantees,,1733201125732.5585179eb550a7f04eee9f770b5bea8c.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T04:45:51,359 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T04:45:51,360 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733201151359"}]},"ts":"9223372036854775807"} 2024-12-03T04:45:51,365 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T04:45:51,371 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T04:45:51,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 148 msec 2024-12-03T04:45:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T04:45:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T04:45:51,542 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T04:45:51,553 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=457 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=653 (was 739), ProcessCount=11 (was 11), AvailableMemoryMB=3226 (was 3360) 2024-12-03T04:45:51,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T04:45:51,554 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T04:45:51,554 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T04:45:51,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:51,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:51,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T04:45:51,554 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T04:45:51,554 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=900566320, stopped=false 2024-12-03T04:45:51,555 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=32beccd76b35,38397,1733200986071 2024-12-03T04:45:51,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T04:45:51,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T04:45:51,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:45:51,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:45:51,557 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T04:45:51,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T04:45:51,558 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T04:45:51,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T04:45:51,558 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T04:45:51,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:51,559 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32beccd76b35,43193,1733200986730' ***** 2024-12-03T04:45:51,559 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T04:45:51,559 INFO [RS:0;32beccd76b35:43193 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T04:45:51,560 INFO [RS:0;32beccd76b35:43193 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T04:45:51,560 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T04:45:51,560 INFO [RS:0;32beccd76b35:43193 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T04:45:51,560 INFO [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(959): stopping server 32beccd76b35,43193,1733200986730 2024-12-03T04:45:51,560 INFO [RS:0;32beccd76b35:43193 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T04:45:51,560 INFO [RS:0;32beccd76b35:43193 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;32beccd76b35:43193. 2024-12-03T04:45:51,560 DEBUG [RS:0;32beccd76b35:43193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T04:45:51,560 DEBUG [RS:0;32beccd76b35:43193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:51,560 INFO [RS:0;32beccd76b35:43193 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T04:45:51,560 INFO [RS:0;32beccd76b35:43193 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T04:45:51,560 INFO [RS:0;32beccd76b35:43193 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T04:45:51,561 INFO [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T04:45:51,561 INFO [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T04:45:51,562 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T04:45:51,562 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T04:45:51,562 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T04:45:51,562 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T04:45:51,562 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T04:45:51,562 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T04:45:51,562 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T04:45:51,562 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-03T04:45:51,593 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/info/c5608b873799408e893f680a53799c00 is 95, key is TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533./info:/1733201015969/DeleteFamily/seqid=0 2024-12-03T04:45:51,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742470_1646 (size=6514) 2024-12-03T04:45:51,598 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/info/c5608b873799408e893f680a53799c00 2024-12-03T04:45:51,616 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/ns/480c80fe862746038e285bdee8447ef7 is 93, key is TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533./ns:/1733201015912/DeleteFamily/seqid=0 2024-12-03T04:45:51,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742471_1647 (size=5906) 2024-12-03T04:45:51,649 INFO [regionserver/32beccd76b35:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T04:45:51,762 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T04:45:51,962 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T04:45:52,024 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/ns/480c80fe862746038e285bdee8447ef7 2024-12-03T04:45:52,042 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/rep_barrier/9e90767e559d4ae487d82d80aba5469e is 102, key is TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533./rep_barrier:/1733201015912/DeleteFamily/seqid=0 2024-12-03T04:45:52,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742472_1648 (size=6025) 2024-12-03T04:45:52,139 INFO [regionserver/32beccd76b35:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T04:45:52,139 INFO [regionserver/32beccd76b35:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T04:45:52,163 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T04:45:52,363 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T04:45:52,445 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/rep_barrier/9e90767e559d4ae487d82d80aba5469e 2024-12-03T04:45:52,464 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/table/6378ada480664657905483a537805eea is 96, key is TestAcidGuarantees,,1733200990336.2444c4148e3a1c831bc6f992044b0533./table:/1733201015912/DeleteFamily/seqid=0 2024-12-03T04:45:52,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742473_1649 (size=5936) 2024-12-03T04:45:52,563 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T04:45:52,763 DEBUG [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T04:45:52,868 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/table/6378ada480664657905483a537805eea 2024-12-03T04:45:52,871 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/info/c5608b873799408e893f680a53799c00 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/info/c5608b873799408e893f680a53799c00 2024-12-03T04:45:52,874 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/info/c5608b873799408e893f680a53799c00, entries=12, sequenceid=90, filesize=6.4 K 2024-12-03T04:45:52,875 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/ns/480c80fe862746038e285bdee8447ef7 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/ns/480c80fe862746038e285bdee8447ef7 2024-12-03T04:45:52,878 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/ns/480c80fe862746038e285bdee8447ef7, entries=8, sequenceid=90, filesize=5.8 K 2024-12-03T04:45:52,878 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/rep_barrier/9e90767e559d4ae487d82d80aba5469e as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/rep_barrier/9e90767e559d4ae487d82d80aba5469e 2024-12-03T04:45:52,881 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/rep_barrier/9e90767e559d4ae487d82d80aba5469e, entries=6, sequenceid=90, filesize=5.9 K 2024-12-03T04:45:52,882 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/.tmp/table/6378ada480664657905483a537805eea as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/table/6378ada480664657905483a537805eea 2024-12-03T04:45:52,885 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/table/6378ada480664657905483a537805eea, entries=7, sequenceid=90, filesize=5.8 K 2024-12-03T04:45:52,886 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1324ms, sequenceid=90, compaction requested=false 2024-12-03T04:45:52,900 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-03T04:45:52,901 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T04:45:52,901 INFO [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T04:45:52,901 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733201151561Running coprocessor pre-close hooks at 1733201151562 (+1 ms)Disabling compacts and flushes for region at 1733201151562Disabling writes for close at 1733201151562Obtaining lock to block concurrent updates at 1733201151562Preparing flush snapshotting stores in 1588230740 at 1733201151562Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733201151563 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733201151563Flushing 1588230740/info: creating writer at 1733201151564 (+1 ms)Flushing 1588230740/info: appending metadata at 1733201151592 (+28 ms)Flushing 1588230740/info: closing flushed file at 1733201151592Flushing 1588230740/ns: creating writer at 1733201151601 (+9 ms)Flushing 1588230740/ns: appending metadata at 1733201151616 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733201151616Flushing 1588230740/rep_barrier: creating writer at 1733201152027 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733201152041 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733201152041Flushing 1588230740/table: creating writer at 1733201152449 (+408 ms)Flushing 1588230740/table: appending metadata at 1733201152464 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733201152464Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@309ad136: reopening flushed file at 1733201152870 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c049cd6: reopening flushed file at 1733201152875 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@851994f: reopening flushed file at 1733201152878 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b2c38c0: reopening flushed file at 1733201152882 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1324ms, sequenceid=90, compaction requested=false at 1733201152886 (+4 ms)Writing region close event to WAL at 1733201152897 (+11 ms)Running coprocessor post-close hooks at 1733201152900 (+3 ms)Closed at 1733201152901 (+1 ms) 2024-12-03T04:45:52,901 DEBUG [RS_CLOSE_META-regionserver/32beccd76b35:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T04:45:52,964 INFO [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(976): stopping server 32beccd76b35,43193,1733200986730; all regions closed. 2024-12-03T04:45:52,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741834_1010 (size=25344) 2024-12-03T04:45:52,970 DEBUG [RS:0;32beccd76b35:43193 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/oldWALs 2024-12-03T04:45:52,970 INFO [RS:0;32beccd76b35:43193 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32beccd76b35%2C43193%2C1733200986730.meta:.meta(num 1733200989867) 2024-12-03T04:45:52,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741833_1009 (size=17657906) 2024-12-03T04:45:52,974 DEBUG [RS:0;32beccd76b35:43193 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/oldWALs 2024-12-03T04:45:52,974 INFO [RS:0;32beccd76b35:43193 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32beccd76b35%2C43193%2C1733200986730:(num 1733200989290) 2024-12-03T04:45:52,974 DEBUG [RS:0;32beccd76b35:43193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T04:45:52,974 INFO [RS:0;32beccd76b35:43193 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T04:45:52,974 INFO [RS:0;32beccd76b35:43193 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T04:45:52,975 INFO [RS:0;32beccd76b35:43193 {}] hbase.ChoreService(370): Chore service for: regionserver/32beccd76b35:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T04:45:52,975 INFO [RS:0;32beccd76b35:43193 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T04:45:52,975 INFO [regionserver/32beccd76b35:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T04:45:52,976 INFO [RS:0;32beccd76b35:43193 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43193 2024-12-03T04:45:52,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T04:45:52,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32beccd76b35,43193,1733200986730 2024-12-03T04:45:52,980 INFO [RS:0;32beccd76b35:43193 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T04:45:52,982 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32beccd76b35,43193,1733200986730] 2024-12-03T04:45:52,984 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32beccd76b35,43193,1733200986730 already deleted, retry=false 2024-12-03T04:45:52,984 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32beccd76b35,43193,1733200986730 expired; onlineServers=0 2024-12-03T04:45:52,984 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '32beccd76b35,38397,1733200986071' ***** 2024-12-03T04:45:52,984 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T04:45:52,984 INFO [M:0;32beccd76b35:38397 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T04:45:52,984 INFO [M:0;32beccd76b35:38397 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T04:45:52,984 DEBUG [M:0;32beccd76b35:38397 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T04:45:52,985 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T04:45:52,985 DEBUG [M:0;32beccd76b35:38397 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T04:45:52,985 DEBUG [master/32beccd76b35:0:becomeActiveMaster-HFileCleaner.large.0-1733200988993 {}] cleaner.HFileCleaner(306): Exit Thread[master/32beccd76b35:0:becomeActiveMaster-HFileCleaner.large.0-1733200988993,5,FailOnTimeoutGroup] 2024-12-03T04:45:52,985 INFO [M:0;32beccd76b35:38397 {}] hbase.ChoreService(370): Chore service for: master/32beccd76b35:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T04:45:52,985 INFO [M:0;32beccd76b35:38397 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T04:45:52,985 DEBUG [M:0;32beccd76b35:38397 {}] master.HMaster(1795): Stopping service threads 2024-12-03T04:45:52,985 INFO [M:0;32beccd76b35:38397 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T04:45:52,985 INFO [M:0;32beccd76b35:38397 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T04:45:52,985 ERROR [M:0;32beccd76b35:38397 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44839,5,PEWorkerGroup] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:44839 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44839,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-03T04:45:52,985 DEBUG [master/32beccd76b35:0:becomeActiveMaster-HFileCleaner.small.0-1733200988998 {}] cleaner.HFileCleaner(306): Exit Thread[master/32beccd76b35:0:becomeActiveMaster-HFileCleaner.small.0-1733200988998,5,FailOnTimeoutGroup] 2024-12-03T04:45:52,986 INFO [M:0;32beccd76b35:38397 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T04:45:52,986 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T04:45:52,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T04:45:52,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T04:45:52,987 DEBUG [M:0;32beccd76b35:38397 {}] zookeeper.ZKUtil(347): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T04:45:52,987 WARN [M:0;32beccd76b35:38397 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T04:45:52,988 INFO [M:0;32beccd76b35:38397 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/.lastflushedseqids 2024-12-03T04:45:52,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742474_1650 (size=72) 2024-12-03T04:45:52,997 INFO [M:0;32beccd76b35:38397 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T04:45:52,997 INFO [M:0;32beccd76b35:38397 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T04:45:52,998 DEBUG [M:0;32beccd76b35:38397 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T04:45:52,998 INFO [M:0;32beccd76b35:38397 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:45:52,998 DEBUG [M:0;32beccd76b35:38397 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:45:52,998 DEBUG [M:0;32beccd76b35:38397 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T04:45:52,998 DEBUG [M:0;32beccd76b35:38397 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:45:52,998 INFO [M:0;32beccd76b35:38397 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=767.86 KB heapSize=940.13 KB 2024-12-03T04:45:53,013 DEBUG [M:0;32beccd76b35:38397 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b457ec85ba49417c93f86613fb3be966 is 82, key is hbase:meta,,1/info:regioninfo/1733200989964/Put/seqid=0 2024-12-03T04:45:53,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742475_1651 (size=5672) 2024-12-03T04:45:53,018 INFO [M:0;32beccd76b35:38397 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2131 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b457ec85ba49417c93f86613fb3be966 2024-12-03T04:45:53,037 DEBUG [M:0;32beccd76b35:38397 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8d3a3bdc49254e01bfd07715c93a18f5 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1733201017610/Put/seqid=0 2024-12-03T04:45:53,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742476_1652 (size=44035) 2024-12-03T04:45:53,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T04:45:53,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43193-0x101516731320001, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T04:45:53,083 INFO [RS:0;32beccd76b35:43193 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T04:45:53,083 INFO [RS:0;32beccd76b35:43193 {}] regionserver.HRegionServer(1031): Exiting; stopping=32beccd76b35,43193,1733200986730; zookeeper connection closed. 2024-12-03T04:45:53,084 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@178ce89c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@178ce89c 2024-12-03T04:45:53,084 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T04:45:53,441 INFO [M:0;32beccd76b35:38397 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=767.30 KB at sequenceid=2131 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8d3a3bdc49254e01bfd07715c93a18f5 2024-12-03T04:45:53,445 INFO [M:0;32beccd76b35:38397 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8d3a3bdc49254e01bfd07715c93a18f5 2024-12-03T04:45:53,465 DEBUG [M:0;32beccd76b35:38397 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/426d9ab628fe4343aa41e241a387fe91 is 69, key is 32beccd76b35,43193,1733200986730/rs:state/1733200989044/Put/seqid=0 2024-12-03T04:45:53,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073742477_1653 (size=5156) 2024-12-03T04:45:53,871 INFO [M:0;32beccd76b35:38397 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2131 (bloomFilter=true), to=hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/426d9ab628fe4343aa41e241a387fe91 2024-12-03T04:45:53,874 DEBUG [M:0;32beccd76b35:38397 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b457ec85ba49417c93f86613fb3be966 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b457ec85ba49417c93f86613fb3be966 2024-12-03T04:45:53,877 INFO [M:0;32beccd76b35:38397 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b457ec85ba49417c93f86613fb3be966, entries=8, sequenceid=2131, filesize=5.5 K 2024-12-03T04:45:53,877 DEBUG [M:0;32beccd76b35:38397 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8d3a3bdc49254e01bfd07715c93a18f5 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8d3a3bdc49254e01bfd07715c93a18f5 2024-12-03T04:45:53,880 INFO [M:0;32beccd76b35:38397 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8d3a3bdc49254e01bfd07715c93a18f5 2024-12-03T04:45:53,880 INFO [M:0;32beccd76b35:38397 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8d3a3bdc49254e01bfd07715c93a18f5, entries=162, sequenceid=2131, filesize=43.0 K 2024-12-03T04:45:53,881 DEBUG [M:0;32beccd76b35:38397 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/426d9ab628fe4343aa41e241a387fe91 as hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/426d9ab628fe4343aa41e241a387fe91 2024-12-03T04:45:53,883 INFO [M:0;32beccd76b35:38397 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44839/user/jenkins/test-data/0b6a1ffd-fdad-ae7f-29fa-7cd92623c676/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/426d9ab628fe4343aa41e241a387fe91, entries=1, sequenceid=2131, filesize=5.0 K 2024-12-03T04:45:53,884 INFO [M:0;32beccd76b35:38397 {}] regionserver.HRegion(3140): Finished flush of dataSize ~767.86 KB/786285, heapSize ~939.84 KB/962392, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 886ms, sequenceid=2131, compaction requested=false 2024-12-03T04:45:53,885 INFO [M:0;32beccd76b35:38397 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T04:45:53,885 DEBUG [M:0;32beccd76b35:38397 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733201152998Disabling compacts and flushes for region at 1733201152998Disabling writes for close at 1733201152998Obtaining lock to block concurrent updates at 1733201152998Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733201152998Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=786285, getHeapSize=962632, getOffHeapSize=0, getCellsCount=2318 at 1733201152998Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733201152999 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733201152999Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733201153013 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733201153013Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733201153021 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733201153036 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733201153036Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733201153445 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733201153464 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733201153464Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@970abcf: reopening flushed file at 1733201153874 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12dffa96: reopening flushed file at 1733201153877 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18d99c64: reopening flushed file at 1733201153880 (+3 ms)Finished flush of dataSize ~767.86 KB/786285, heapSize ~939.84 KB/962392, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 886ms, sequenceid=2131, compaction requested=false at 1733201153884 (+4 ms)Writing region close event to WAL at 1733201153885 (+1 ms)Closed at 1733201153885 2024-12-03T04:45:53,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741830_1006 (size=925967) 2024-12-03T04:45:53,888 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T04:45:53,888 INFO [M:0;32beccd76b35:38397 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T04:45:53,888 INFO [M:0;32beccd76b35:38397 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38397 2024-12-03T04:45:53,889 INFO [M:0;32beccd76b35:38397 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T04:45:53,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T04:45:53,991 INFO [M:0;32beccd76b35:38397 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T04:45:53,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38397-0x101516731320000, quorum=127.0.0.1:56849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T04:45:54,011 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T04:45:54,013 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T04:45:54,014 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T04:45:54,014 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T04:45:54,014 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/hadoop.log.dir/,STOPPED} 2024-12-03T04:45:54,018 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T04:45:54,018 WARN [BP-1463157627-172.17.0.3-1733200983164 heartbeating to localhost/127.0.0.1:44839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T04:45:54,018 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T04:45:54,018 WARN [BP-1463157627-172.17.0.3-1733200983164 heartbeating to localhost/127.0.0.1:44839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1463157627-172.17.0.3-1733200983164 (Datanode Uuid 0b750991-f791-487f-8920-7516e0a87234) service to localhost/127.0.0.1:44839 2024-12-03T04:45:54,021 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/cluster_42193a68-94b8-0488-849e-db0fb1205e79/data/data1/current/BP-1463157627-172.17.0.3-1733200983164 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T04:45:54,021 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/cluster_42193a68-94b8-0488-849e-db0fb1205e79/data/data2/current/BP-1463157627-172.17.0.3-1733200983164 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T04:45:54,022 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T04:45:54,031 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T04:45:54,032 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T04:45:54,032 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T04:45:54,032 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T04:45:54,032 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3fce05d2-1319-9cf0-dcbd-74d0c6159deb/hadoop.log.dir/,STOPPED} 2024-12-03T04:45:54,048 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T04:45:54,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down